v0.3.0 released, with very significant improvements!

magneticod:
* Stalled DisposablePeers are shut down after 120 seconds (fixes #47 and #38)
* Improved performance, thanks to @ngosang, @ad-m, and all other contributors (fixes #48)
* Default logging level is now set to INFO (since many users would freak out after seeing "peer failed" messages...)

magneticow:
* Search speed improved A LOT, like "A  L O T"! (fixes #8)
  (for the curious, the problem was that magneticow initialised database *every* *single* *time* a request is made because
  "the global application context" (`flask.g`) didn't work for some reason I don't know. Now it's blazing fast!
* A bit of logging added to assist the user. This is especially important as magneticow now takes a bit of time to initialize itself
  (as reasonably expected) and will not be able to handle any requests until it's complete.
* A faster but possibly less accurate (and not-guaranteed behaviour dependent) calculation of the number of torrents.
* Thousands separator for the torrent count in the homepage.
This commit is contained in:
Bora M. ALPER 2017-04-26 01:47:13 +04:00
parent 9dc1684e75
commit 3b39c99206
10 changed files with 50 additions and 33 deletions

View File

@ -12,4 +12,4 @@
#
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
# <http://www.gnu.org/licenses/>.
__version__ = (0, 2, 0)
__version__ = (0, 3, 0)

View File

@ -55,7 +55,7 @@ complete_info_hashes = set()
def main():
global complete_info_hashes, database, node, peers, selector
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)-8s %(message)s")
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s")
logging.info("magneticod v%d.%d.%d started", *__version__)
arguments = parse_cmdline_arguments()
@ -132,6 +132,9 @@ def on_peer_error(peer: bittorrent.DisposablePeer, info_hash: dht.InfoHash) -> N
selector.unregister(peer)
# TODO:
# Consider whether time.monotonic() is a good choice. Maybe we should use CLOCK_MONOTONIC_RAW as its not affected by NTP
# adjustments, and all we need is how many seconds passed since a certain point in time.
def loop() -> None:
global selector, node, peers
@ -146,6 +149,9 @@ def loop() -> None:
logging.warning("Belated TICK! (Δ = %d)", delta)
node.on_tick()
for peer_list in peers.values():
for peer in peer_list:
peer.on_tick()
t0 = time.monotonic()

View File

@ -17,7 +17,6 @@ import logging
import hashlib
import math
import socket
import random
import typing
import os
@ -57,6 +56,9 @@ class DisposablePeer:
# To prevent double shutdown
self.__shutdown = False
# After 120 ticks passed, a peer should report an error and shut itself down due to being stall.
self.__ticks_passed = 0
# Send the BitTorrent handshake message (0x13 = 19 in decimal, the length of the handshake message)
self.__outgoing_buffer += b"\x13BitTorrent protocol%s%s%s" % (
b"\x00\x00\x00\x00\x00\x10\x00\x01",
@ -72,6 +74,13 @@ class DisposablePeer:
def when_metadata_found(info_hash: InfoHash, metadata: bytes) -> None:
raise NotImplementedError()
def on_tick(self):
self.__ticks_passed += 1
if self.__ticks_passed == 120:
logging.debug("Peer failed to fetch metadata in time for info hash %s!", self.__info_hash.hex())
self.when_error()
def on_receivable(self) -> None:
while True:
try:
@ -211,6 +220,8 @@ class DisposablePeer:
except MemoryError:
logging.exception("Could not allocate %.1f KiB for the metadata!", metadata_size / 1024)
self.when_error()
return
self.__metadata_size = metadata_size
self.__ext_handshake_complete = True

View File

@ -16,7 +16,6 @@ import array
import collections
import zlib
import logging
import random
import socket
import typing
import os
@ -54,7 +53,7 @@ class SybilNode:
# stop; but until then, the total number of neighbours might exceed the threshold).
self.__n_max_neighbours = 2000
logging.debug("SybilNode %s on %s initialized!", self.__true_id.hex().upper(), address)
logging.info("SybilNode %s on %s initialized!", self.__true_id.hex().upper(), address)
@staticmethod
def when_peer_found(info_hash: InfoHash, peer_addr: PeerAddress) -> None:
@ -165,7 +164,7 @@ class SybilNode:
return
# appendleft to prioritise GET_PEERS responses as they are the most fruitful ones!
self.__outgoing_queue.append((addr, bencode.dumps({
self.__outgoing_queue.appendleft((addr, bencode.dumps({
b"y": b"r",
b"t": transaction_id,
b"r": {

View File

@ -119,7 +119,7 @@ class Database:
self.__pending_files
)
cur.execute("COMMIT;")
logging.debug("%d metadata (%d files) are committed to the database.",
logging.info("%d metadata (%d files) are committed to the database.",
len(self.__pending_metadata), len(self.__pending_files))
self.__pending_metadata.clear()
self.__pending_files.clear()

View File

@ -8,7 +8,7 @@ def read_file(path):
setup(
name="magneticod",
version="0.2.0",
version="0.3.0",
description="Autonomous BitTorrent DHT crawler and metadata fetcher.",
long_description=read_file("README.rst"),
url="http://magnetico.org",

View File

@ -14,6 +14,7 @@
# <http://www.gnu.org/licenses/>.
import argparse
import logging
import sys
import textwrap
@ -23,15 +24,22 @@ from magneticow import magneticow
def main() -> int:
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s")
arguments = parse_args()
magneticow.app.arguments = arguments
http_server = gevent.wsgi.WSGIServer(("", arguments.port), magneticow.app)
magneticow.initialize_magneticod_db()
try:
logging.info("magneticow is ready to serve!")
http_server.serve_forever()
except KeyboardInterrupt:
return 0
finally:
magneticow.close_db()
return 1
@ -71,4 +79,4 @@ def parse_args() -> dict:
return parser.parse_args(sys.argv[1:])
if __name__ == "__main__":
sys.exit(main())
sys.exit(main())

View File

@ -15,6 +15,7 @@
import collections
import functools
from datetime import datetime
import logging
import sqlite3
import os
@ -31,6 +32,10 @@ Torrent = collections.namedtuple("torrent", ["info_hash", "name", "size", "disco
app = flask.Flask(__name__)
app.config.from_object(__name__)
# TODO: We should have been able to use flask.g but it does NOT persist across different requests so we resort back to
# this. Investigate the cause and fix it (I suspect of Gevent).
magneticod_db = None
# Adapted from: http://flask.pocoo.org/snippets/8/
# (c) Copyright 2010 - 2017 by Armin Ronacher
@ -67,10 +72,8 @@ def requires_auth(f):
@app.route("/")
@requires_auth
def home_page():
magneticod_db = get_magneticod_db()
with magneticod_db:
cur = magneticod_db.execute("SELECT count() FROM torrents ;")
cur = magneticod_db.execute("SELECT MAX(ROWID) FROM torrents ;")
n_torrents = cur.fetchone()[0]
return flask.render_template("homepage.html", n_torrents=n_torrents)
@ -88,8 +91,6 @@ def torrents():
def search_torrents():
magneticod_db = get_magneticod_db()
search = flask.request.args["search"]
page = int(flask.request.args.get("page", 0))
@ -128,8 +129,6 @@ def search_torrents():
def newest_torrents():
magneticod_db = get_magneticod_db()
page = int(flask.request.args.get("page", 0))
context = {
@ -162,8 +161,6 @@ def newest_torrents():
@app.route("/torrents/<info_hash>/", defaults={"name": None})
@requires_auth
def torrent_redirect(**kwargs):
magnetico_db = get_magneticod_db()
try:
info_hash = bytes.fromhex(kwargs["info_hash"])
assert len(info_hash) == 20
@ -183,7 +180,6 @@ def torrent_redirect(**kwargs):
@app.route("/torrents/<info_hash>/<name>")
@requires_auth
def torrent(**kwargs):
magneticod_db = get_magneticod_db()
context = {}
try:
@ -210,13 +206,13 @@ def torrent(**kwargs):
return flask.render_template("torrent.html", **context)
def get_magneticod_db():
""" Opens a new database connection if there is none yet for the current application context. """
if hasattr(flask.g, "magneticod_db"):
return flask.g.magneticod_db
def initialize_magneticod_db() -> None:
global magneticod_db
logging.info("Connecting to magneticod's database...")
magneticod_db_path = os.path.join(appdirs.user_data_dir("magneticod"), "database.sqlite3")
magneticod_db = flask.g.magneticod_db = sqlite3.connect(magneticod_db_path, isolation_level=None)
magneticod_db = sqlite3.connect(magneticod_db_path, isolation_level=None)
with magneticod_db:
magneticod_db.execute("PRAGMA journal_mode=WAL;")
@ -232,11 +228,8 @@ def get_magneticod_db():
magneticod_db.create_function("rank", 1, utils.rank)
return magneticod_db
@app.teardown_appcontext
def close_magneticod_db(error):
""" Closes the database again at the end of the request. """
if hasattr(flask.g, "magneticod_db"):
flask.g.magneticod_db.close()
def close_db() -> None:
logging.info("Closing magneticod database...")
if magneticod_db is not None:
magneticod_db.close()

View File

@ -17,7 +17,7 @@
</main>
<footer>
Now with {{n_torrents}} torrents available !
Now with {{ "{:,}".format(n_torrents) }} torrents available !
</footer>
</body>
</html>

View File

@ -8,7 +8,7 @@ def read_file(path):
setup(
name="magneticow",
version="0.2.0",
version="0.3.0",
description="Lightweight web interface for magnetico.",
long_description=read_file("README.rst"),
url="http://magnetico.org",