v0.3.0 released, with very significant improvements!

magneticod:
* Stalled DisposablePeers are shut down after 120 seconds (fixes #47 and #38)
* Improved performance, thanks to @ngosang, @ad-m, and all other contributors (fixes #48)
* Default logging level is now set to INFO (since many users would freak out after seeing "peer failed" messages...)

magneticow:
* Search speed improved A LOT, like "A  L O T"! (fixes #8)
  (for the curious, the problem was that magneticow initialised database *every* *single* *time* a request is made because
  "the global application context" (`flask.g`) didn't work for some reason I don't know. Now it's blazing fast!
* A bit of logging added to assist the user. This is especially important as magneticow now takes a bit of time to initialize itself
  (as reasonably expected) and will not be able to handle any requests until it's complete.
* A faster but possibly less accurate (and not-guaranteed behaviour dependent) calculation of the number of torrents.
* Thousands separator for the torrent count in the homepage.

magneticod/magneticod/__init__.py

@@ -12,4 +12,4 @@
 #
 # You should have received a copy of the GNU Affero General Public License along with this program.  If not, see
 # <http://www.gnu.org/licenses/>.
-__version__ = (0, 2, 0)
+__version__ = (0, 3, 0)

magneticod/magneticod/__main__.py

@@ -55,7 +55,7 @@ complete_info_hashes = set()
 def main():
     global complete_info_hashes, database, node, peers, selector
 
-    logging.basicConfig(level=logging.DEBUG, format="%(asctime)s  %(levelname)-8s  %(message)s")
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(levelname)-8s  %(message)s")
     logging.info("magneticod v%d.%d.%d started", *__version__)
 
     arguments = parse_cmdline_arguments()
@@ -132,6 +132,9 @@ def on_peer_error(peer: bittorrent.DisposablePeer, info_hash: dht.InfoHash) -> N
     selector.unregister(peer)
 
 
+# TODO:
+# Consider whether time.monotonic() is a good choice. Maybe we should use CLOCK_MONOTONIC_RAW as its not affected by NTP
+# adjustments, and all we need is how many seconds passed since a certain point in time.
 def loop() -> None:
     global selector, node, peers
 
@@ -146,6 +149,9 @@ def loop() -> None:
                 logging.warning("Belated TICK! (Δ = %d)", delta)
 
             node.on_tick()
+            for peer_list in peers.values():
+                for peer in peer_list:
+                    peer.on_tick()
 
             t0 = time.monotonic()
 

magneticod/magneticod/bittorrent.py

@@ -17,7 +17,6 @@ import logging
 import hashlib
 import math
 import socket
-import random
 import typing
 import os
 
@@ -57,6 +56,9 @@ class DisposablePeer:
         # To prevent double shutdown
         self.__shutdown = False
 
+        # After 120 ticks passed, a peer should report an error and shut itself down due to being stall.
+        self.__ticks_passed = 0
+
         # Send the BitTorrent handshake message (0x13 = 19 in decimal, the length of the handshake message)
         self.__outgoing_buffer += b"\x13BitTorrent protocol%s%s%s" % (
             b"\x00\x00\x00\x00\x00\x10\x00\x01",
@@ -72,6 +74,13 @@ class DisposablePeer:
     def when_metadata_found(info_hash: InfoHash, metadata: bytes) -> None:
         raise NotImplementedError()
 
+    def on_tick(self):
+        self.__ticks_passed += 1
+
+        if self.__ticks_passed == 120:
+            logging.debug("Peer failed to fetch metadata in time for info hash %s!", self.__info_hash.hex())
+            self.when_error()
+
     def on_receivable(self) -> None:
         while True:
             try:
@@ -211,6 +220,8 @@ class DisposablePeer:
         except MemoryError:
             logging.exception("Could not allocate %.1f KiB for the metadata!", metadata_size / 1024)
             self.when_error()
+            return
+
         self.__metadata_size = metadata_size
         self.__ext_handshake_complete = True
 

magneticod/magneticod/dht.py

@@ -16,7 +16,6 @@ import array
 import collections
 import zlib
 import logging
-import random
 import socket
 import typing
 import os
@@ -54,7 +53,7 @@ class SybilNode:
         # stop; but until then, the total number of neighbours might exceed the threshold).
         self.__n_max_neighbours = 2000
 
-        logging.debug("SybilNode %s on %s initialized!", self.__true_id.hex().upper(), address)
+        logging.info("SybilNode %s on %s initialized!", self.__true_id.hex().upper(), address)
 
     @staticmethod
     def when_peer_found(info_hash: InfoHash, peer_addr: PeerAddress) -> None:
@@ -165,7 +164,7 @@ class SybilNode:
             return
 
         # appendleft to prioritise GET_PEERS responses as they are the most fruitful ones!
-        self.__outgoing_queue.append((addr, bencode.dumps({
+        self.__outgoing_queue.appendleft((addr, bencode.dumps({
             b"y": b"r",
             b"t": transaction_id,
             b"r": {

magneticod/magneticod/persistence.py

@@ -119,7 +119,7 @@ class Database:
                 self.__pending_files
             )
             cur.execute("COMMIT;")
-            logging.debug("%d metadata (%d files) are committed to the database.",
+            logging.info("%d metadata (%d files) are committed to the database.",
                           len(self.__pending_metadata), len(self.__pending_files))
             self.__pending_metadata.clear()
             self.__pending_files.clear()

magneticod/setup.py

@@ -8,7 +8,7 @@ def read_file(path):
 
 setup(
     name="magneticod",
-    version="0.2.0",
+    version="0.3.0",
     description="Autonomous BitTorrent DHT crawler and metadata fetcher.",
     long_description=read_file("README.rst"),
     url="http://magnetico.org",

magneticow/magneticow/__main__.py

@@ -14,6 +14,7 @@
 # <http://www.gnu.org/licenses/>.
 
 import argparse
+import logging
 import sys
 import textwrap
 
@@ -23,15 +24,22 @@ from magneticow import magneticow
 
 
 def main() -> int:
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s  %(levelname)-8s  %(message)s")
+
     arguments = parse_args()
     magneticow.app.arguments = arguments
 
     http_server = gevent.wsgi.WSGIServer(("", arguments.port), magneticow.app)
 
+    magneticow.initialize_magneticod_db()
+
     try:
+        logging.info("magneticow is ready to serve!")
         http_server.serve_forever()
     except KeyboardInterrupt:
         return 0
+    finally:
+        magneticow.close_db()
 
     return 1
 
@@ -71,4 +79,4 @@ def parse_args() -> dict:
     return parser.parse_args(sys.argv[1:])
 
 if __name__ == "__main__":
-    sys.exit(main())
+    sys.exit(main())

magneticow/magneticow/magneticow.py

@@ -15,6 +15,7 @@
 import collections
 import functools
 from datetime import datetime
+import logging
 import sqlite3
 import os
 
@@ -31,6 +32,10 @@ Torrent = collections.namedtuple("torrent", ["info_hash", "name", "size", "disco
 app = flask.Flask(__name__)
 app.config.from_object(__name__)
 
+# TODO: We should have been able to use flask.g but it does NOT persist across different requests so we resort back to
+# this. Investigate the cause and fix it (I suspect of Gevent).
+magneticod_db = None
+
 
 # Adapted from: http://flask.pocoo.org/snippets/8/
 # (c) Copyright 2010 - 2017 by Armin Ronacher
@@ -67,10 +72,8 @@ def requires_auth(f):
 @app.route("/")
 @requires_auth
 def home_page():
-    magneticod_db = get_magneticod_db()
-
     with magneticod_db:
-        cur = magneticod_db.execute("SELECT count() FROM torrents ;")
+        cur = magneticod_db.execute("SELECT MAX(ROWID) FROM torrents ;")
         n_torrents = cur.fetchone()[0]
 
     return flask.render_template("homepage.html", n_torrents=n_torrents)
@@ -88,8 +91,6 @@ def torrents():
 
 
 def search_torrents():
-    magneticod_db = get_magneticod_db()
-
     search = flask.request.args["search"]
     page = int(flask.request.args.get("page", 0))
 
@@ -128,8 +129,6 @@ def search_torrents():
 
 
 def newest_torrents():
-    magneticod_db = get_magneticod_db()
-
     page = int(flask.request.args.get("page", 0))
 
     context = {
@@ -162,8 +161,6 @@ def newest_torrents():
 @app.route("/torrents/<info_hash>/", defaults={"name": None})
 @requires_auth
 def torrent_redirect(**kwargs):
-    magnetico_db = get_magneticod_db()
-
     try:
         info_hash = bytes.fromhex(kwargs["info_hash"])
         assert len(info_hash) == 20
@@ -183,7 +180,6 @@ def torrent_redirect(**kwargs):
 @app.route("/torrents/<info_hash>/<name>")
 @requires_auth
 def torrent(**kwargs):
-    magneticod_db = get_magneticod_db()
     context = {}
 
     try:
@@ -210,13 +206,13 @@ def torrent(**kwargs):
     return flask.render_template("torrent.html", **context)
 
 
-def get_magneticod_db():
+def initialize_magneticod_db() -> None:
-    """ Opens a new database connection if there is none yet for the current application context. """
+    global magneticod_db
-    if hasattr(flask.g, "magneticod_db"):
+
-        return flask.g.magneticod_db
+    logging.info("Connecting to magneticod's database...")
 
     magneticod_db_path = os.path.join(appdirs.user_data_dir("magneticod"), "database.sqlite3")
-    magneticod_db = flask.g.magneticod_db = sqlite3.connect(magneticod_db_path, isolation_level=None)
+    magneticod_db = sqlite3.connect(magneticod_db_path, isolation_level=None)
 
     with magneticod_db:
         magneticod_db.execute("PRAGMA journal_mode=WAL;")
@@ -232,11 +228,8 @@ def get_magneticod_db():
 
     magneticod_db.create_function("rank", 1, utils.rank)
 
-    return magneticod_db
 
-
+def close_db() -> None:
-@app.teardown_appcontext
+    logging.info("Closing magneticod database...")
-def close_magneticod_db(error):
+    if magneticod_db is not None:
-    """ Closes the database again at the end of the request. """
+        magneticod_db.close()
-    if hasattr(flask.g, "magneticod_db"):
-        flask.g.magneticod_db.close()

magneticow/magneticow/templates/homepage.html

@@ -17,7 +17,7 @@
 </main>
 
 <footer>
-  Now with {{n_torrents}} torrents available !
+  Now with {{ "{:,}".format(n_torrents) }} torrents available !
 </footer>
 </body>
 </html>

magneticow/setup.py

@@ -8,7 +8,7 @@ def read_file(path):
 
 setup(
     name="magneticow",
-    version="0.2.0",
+    version="0.3.0",
     description="Lightweight web interface for magnetico.",
     long_description=read_file("README.rst"),
     url="http://magnetico.org",