From 3b39c992068867e505c22f84136b6817cd361887 Mon Sep 17 00:00:00 2001 From: "Bora M. ALPER" Date: Wed, 26 Apr 2017 01:47:13 +0400 Subject: [PATCH] v0.3.0 released, with very significant improvements! magneticod: * Stalled DisposablePeers are shut down after 120 seconds (fixes #47 and #38) * Improved performance, thanks to @ngosang, @ad-m, and all other contributors (fixes #48) * Default logging level is now set to INFO (since many users would freak out after seeing "peer failed" messages...) magneticow: * Search speed improved A LOT, like "A L O T"! (fixes #8) (for the curious, the problem was that magneticow initialised database *every* *single* *time* a request is made because "the global application context" (`flask.g`) didn't work for some reason I don't know. Now it's blazing fast! * A bit of logging added to assist the user. This is especially important as magneticow now takes a bit of time to initialize itself (as reasonably expected) and will not be able to handle any requests until it's complete. * A faster but possibly less accurate (and not-guaranteed behaviour dependent) calculation of the number of torrents. * Thousands separator for the torrent count in the homepage. --- magneticod/magneticod/__init__.py | 2 +- magneticod/magneticod/__main__.py | 8 +++- magneticod/magneticod/bittorrent.py | 13 ++++++- magneticod/magneticod/dht.py | 5 +-- magneticod/magneticod/persistence.py | 2 +- magneticod/setup.py | 2 +- magneticow/magneticow/__main__.py | 10 ++++- magneticow/magneticow/magneticow.py | 37 ++++++++----------- magneticow/magneticow/templates/homepage.html | 2 +- magneticow/setup.py | 2 +- 10 files changed, 50 insertions(+), 33 deletions(-) diff --git a/magneticod/magneticod/__init__.py b/magneticod/magneticod/__init__.py index 97e0021..b775b85 100644 --- a/magneticod/magneticod/__init__.py +++ b/magneticod/magneticod/__init__.py @@ -12,4 +12,4 @@ # # You should have received a copy of the GNU Affero General Public License along with this program. If not, see # . -__version__ = (0, 2, 0) +__version__ = (0, 3, 0) diff --git a/magneticod/magneticod/__main__.py b/magneticod/magneticod/__main__.py index 9f2bf73..70d7cc7 100644 --- a/magneticod/magneticod/__main__.py +++ b/magneticod/magneticod/__main__.py @@ -55,7 +55,7 @@ complete_info_hashes = set() def main(): global complete_info_hashes, database, node, peers, selector - logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)-8s %(message)s") + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s") logging.info("magneticod v%d.%d.%d started", *__version__) arguments = parse_cmdline_arguments() @@ -132,6 +132,9 @@ def on_peer_error(peer: bittorrent.DisposablePeer, info_hash: dht.InfoHash) -> N selector.unregister(peer) +# TODO: +# Consider whether time.monotonic() is a good choice. Maybe we should use CLOCK_MONOTONIC_RAW as its not affected by NTP +# adjustments, and all we need is how many seconds passed since a certain point in time. def loop() -> None: global selector, node, peers @@ -146,6 +149,9 @@ def loop() -> None: logging.warning("Belated TICK! (Δ = %d)", delta) node.on_tick() + for peer_list in peers.values(): + for peer in peer_list: + peer.on_tick() t0 = time.monotonic() diff --git a/magneticod/magneticod/bittorrent.py b/magneticod/magneticod/bittorrent.py index f499d0b..9e4ac54 100644 --- a/magneticod/magneticod/bittorrent.py +++ b/magneticod/magneticod/bittorrent.py @@ -17,7 +17,6 @@ import logging import hashlib import math import socket -import random import typing import os @@ -57,6 +56,9 @@ class DisposablePeer: # To prevent double shutdown self.__shutdown = False + # After 120 ticks passed, a peer should report an error and shut itself down due to being stall. + self.__ticks_passed = 0 + # Send the BitTorrent handshake message (0x13 = 19 in decimal, the length of the handshake message) self.__outgoing_buffer += b"\x13BitTorrent protocol%s%s%s" % ( b"\x00\x00\x00\x00\x00\x10\x00\x01", @@ -72,6 +74,13 @@ class DisposablePeer: def when_metadata_found(info_hash: InfoHash, metadata: bytes) -> None: raise NotImplementedError() + def on_tick(self): + self.__ticks_passed += 1 + + if self.__ticks_passed == 120: + logging.debug("Peer failed to fetch metadata in time for info hash %s!", self.__info_hash.hex()) + self.when_error() + def on_receivable(self) -> None: while True: try: @@ -211,6 +220,8 @@ class DisposablePeer: except MemoryError: logging.exception("Could not allocate %.1f KiB for the metadata!", metadata_size / 1024) self.when_error() + return + self.__metadata_size = metadata_size self.__ext_handshake_complete = True diff --git a/magneticod/magneticod/dht.py b/magneticod/magneticod/dht.py index 5fd1559..dfbfb36 100644 --- a/magneticod/magneticod/dht.py +++ b/magneticod/magneticod/dht.py @@ -16,7 +16,6 @@ import array import collections import zlib import logging -import random import socket import typing import os @@ -54,7 +53,7 @@ class SybilNode: # stop; but until then, the total number of neighbours might exceed the threshold). self.__n_max_neighbours = 2000 - logging.debug("SybilNode %s on %s initialized!", self.__true_id.hex().upper(), address) + logging.info("SybilNode %s on %s initialized!", self.__true_id.hex().upper(), address) @staticmethod def when_peer_found(info_hash: InfoHash, peer_addr: PeerAddress) -> None: @@ -165,7 +164,7 @@ class SybilNode: return # appendleft to prioritise GET_PEERS responses as they are the most fruitful ones! - self.__outgoing_queue.append((addr, bencode.dumps({ + self.__outgoing_queue.appendleft((addr, bencode.dumps({ b"y": b"r", b"t": transaction_id, b"r": { diff --git a/magneticod/magneticod/persistence.py b/magneticod/magneticod/persistence.py index f00e547..afd9077 100644 --- a/magneticod/magneticod/persistence.py +++ b/magneticod/magneticod/persistence.py @@ -119,7 +119,7 @@ class Database: self.__pending_files ) cur.execute("COMMIT;") - logging.debug("%d metadata (%d files) are committed to the database.", + logging.info("%d metadata (%d files) are committed to the database.", len(self.__pending_metadata), len(self.__pending_files)) self.__pending_metadata.clear() self.__pending_files.clear() diff --git a/magneticod/setup.py b/magneticod/setup.py index daca224..1219fe4 100644 --- a/magneticod/setup.py +++ b/magneticod/setup.py @@ -8,7 +8,7 @@ def read_file(path): setup( name="magneticod", - version="0.2.0", + version="0.3.0", description="Autonomous BitTorrent DHT crawler and metadata fetcher.", long_description=read_file("README.rst"), url="http://magnetico.org", diff --git a/magneticow/magneticow/__main__.py b/magneticow/magneticow/__main__.py index ebac3c4..acd470d 100644 --- a/magneticow/magneticow/__main__.py +++ b/magneticow/magneticow/__main__.py @@ -14,6 +14,7 @@ # . import argparse +import logging import sys import textwrap @@ -23,15 +24,22 @@ from magneticow import magneticow def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s") + arguments = parse_args() magneticow.app.arguments = arguments http_server = gevent.wsgi.WSGIServer(("", arguments.port), magneticow.app) + magneticow.initialize_magneticod_db() + try: + logging.info("magneticow is ready to serve!") http_server.serve_forever() except KeyboardInterrupt: return 0 + finally: + magneticow.close_db() return 1 @@ -71,4 +79,4 @@ def parse_args() -> dict: return parser.parse_args(sys.argv[1:]) if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/magneticow/magneticow/magneticow.py b/magneticow/magneticow/magneticow.py index 9a25487..ad69fca 100644 --- a/magneticow/magneticow/magneticow.py +++ b/magneticow/magneticow/magneticow.py @@ -15,6 +15,7 @@ import collections import functools from datetime import datetime +import logging import sqlite3 import os @@ -31,6 +32,10 @@ Torrent = collections.namedtuple("torrent", ["info_hash", "name", "size", "disco app = flask.Flask(__name__) app.config.from_object(__name__) +# TODO: We should have been able to use flask.g but it does NOT persist across different requests so we resort back to +# this. Investigate the cause and fix it (I suspect of Gevent). +magneticod_db = None + # Adapted from: http://flask.pocoo.org/snippets/8/ # (c) Copyright 2010 - 2017 by Armin Ronacher @@ -67,10 +72,8 @@ def requires_auth(f): @app.route("/") @requires_auth def home_page(): - magneticod_db = get_magneticod_db() - with magneticod_db: - cur = magneticod_db.execute("SELECT count() FROM torrents ;") + cur = magneticod_db.execute("SELECT MAX(ROWID) FROM torrents ;") n_torrents = cur.fetchone()[0] return flask.render_template("homepage.html", n_torrents=n_torrents) @@ -88,8 +91,6 @@ def torrents(): def search_torrents(): - magneticod_db = get_magneticod_db() - search = flask.request.args["search"] page = int(flask.request.args.get("page", 0)) @@ -128,8 +129,6 @@ def search_torrents(): def newest_torrents(): - magneticod_db = get_magneticod_db() - page = int(flask.request.args.get("page", 0)) context = { @@ -162,8 +161,6 @@ def newest_torrents(): @app.route("/torrents//", defaults={"name": None}) @requires_auth def torrent_redirect(**kwargs): - magnetico_db = get_magneticod_db() - try: info_hash = bytes.fromhex(kwargs["info_hash"]) assert len(info_hash) == 20 @@ -183,7 +180,6 @@ def torrent_redirect(**kwargs): @app.route("/torrents//") @requires_auth def torrent(**kwargs): - magneticod_db = get_magneticod_db() context = {} try: @@ -210,13 +206,13 @@ def torrent(**kwargs): return flask.render_template("torrent.html", **context) -def get_magneticod_db(): - """ Opens a new database connection if there is none yet for the current application context. """ - if hasattr(flask.g, "magneticod_db"): - return flask.g.magneticod_db +def initialize_magneticod_db() -> None: + global magneticod_db + + logging.info("Connecting to magneticod's database...") magneticod_db_path = os.path.join(appdirs.user_data_dir("magneticod"), "database.sqlite3") - magneticod_db = flask.g.magneticod_db = sqlite3.connect(magneticod_db_path, isolation_level=None) + magneticod_db = sqlite3.connect(magneticod_db_path, isolation_level=None) with magneticod_db: magneticod_db.execute("PRAGMA journal_mode=WAL;") @@ -232,11 +228,8 @@ def get_magneticod_db(): magneticod_db.create_function("rank", 1, utils.rank) - return magneticod_db - -@app.teardown_appcontext -def close_magneticod_db(error): - """ Closes the database again at the end of the request. """ - if hasattr(flask.g, "magneticod_db"): - flask.g.magneticod_db.close() +def close_db() -> None: + logging.info("Closing magneticod database...") + if magneticod_db is not None: + magneticod_db.close() diff --git a/magneticow/magneticow/templates/homepage.html b/magneticow/magneticow/templates/homepage.html index 475a1f0..0ed8648 100644 --- a/magneticow/magneticow/templates/homepage.html +++ b/magneticow/magneticow/templates/homepage.html @@ -17,7 +17,7 @@ diff --git a/magneticow/setup.py b/magneticow/setup.py index 4066bec..0fd87c4 100644 --- a/magneticow/setup.py +++ b/magneticow/setup.py @@ -8,7 +8,7 @@ def read_file(path): setup( name="magneticow", - version="0.2.0", + version="0.3.0", description="Lightweight web interface for magnetico.", long_description=read_file("README.rst"), url="http://magnetico.org",