v0.3.0 released, with very significant improvements!
magneticod: * Stalled DisposablePeers are shut down after 120 seconds (fixes #47 and #38) * Improved performance, thanks to @ngosang, @ad-m, and all other contributors (fixes #48) * Default logging level is now set to INFO (since many users would freak out after seeing "peer failed" messages...) magneticow: * Search speed improved A LOT, like "A L O T"! (fixes #8) (for the curious, the problem was that magneticow initialised database *every* *single* *time* a request is made because "the global application context" (`flask.g`) didn't work for some reason I don't know. Now it's blazing fast! * A bit of logging added to assist the user. This is especially important as magneticow now takes a bit of time to initialize itself (as reasonably expected) and will not be able to handle any requests until it's complete. * A faster but possibly less accurate (and not-guaranteed behaviour dependent) calculation of the number of torrents. * Thousands separator for the torrent count in the homepage.
This commit is contained in:
parent
9dc1684e75
commit
3b39c99206
@ -12,4 +12,4 @@
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License along with this program. If not, see
|
||||
# <http://www.gnu.org/licenses/>.
|
||||
__version__ = (0, 2, 0)
|
||||
__version__ = (0, 3, 0)
|
||||
|
@ -55,7 +55,7 @@ complete_info_hashes = set()
|
||||
def main():
|
||||
global complete_info_hashes, database, node, peers, selector
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)-8s %(message)s")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s")
|
||||
logging.info("magneticod v%d.%d.%d started", *__version__)
|
||||
|
||||
arguments = parse_cmdline_arguments()
|
||||
@ -132,6 +132,9 @@ def on_peer_error(peer: bittorrent.DisposablePeer, info_hash: dht.InfoHash) -> N
|
||||
selector.unregister(peer)
|
||||
|
||||
|
||||
# TODO:
|
||||
# Consider whether time.monotonic() is a good choice. Maybe we should use CLOCK_MONOTONIC_RAW as its not affected by NTP
|
||||
# adjustments, and all we need is how many seconds passed since a certain point in time.
|
||||
def loop() -> None:
|
||||
global selector, node, peers
|
||||
|
||||
@ -146,6 +149,9 @@ def loop() -> None:
|
||||
logging.warning("Belated TICK! (Δ = %d)", delta)
|
||||
|
||||
node.on_tick()
|
||||
for peer_list in peers.values():
|
||||
for peer in peer_list:
|
||||
peer.on_tick()
|
||||
|
||||
t0 = time.monotonic()
|
||||
|
||||
|
@ -17,7 +17,6 @@ import logging
|
||||
import hashlib
|
||||
import math
|
||||
import socket
|
||||
import random
|
||||
import typing
|
||||
import os
|
||||
|
||||
@ -57,6 +56,9 @@ class DisposablePeer:
|
||||
# To prevent double shutdown
|
||||
self.__shutdown = False
|
||||
|
||||
# After 120 ticks passed, a peer should report an error and shut itself down due to being stall.
|
||||
self.__ticks_passed = 0
|
||||
|
||||
# Send the BitTorrent handshake message (0x13 = 19 in decimal, the length of the handshake message)
|
||||
self.__outgoing_buffer += b"\x13BitTorrent protocol%s%s%s" % (
|
||||
b"\x00\x00\x00\x00\x00\x10\x00\x01",
|
||||
@ -72,6 +74,13 @@ class DisposablePeer:
|
||||
def when_metadata_found(info_hash: InfoHash, metadata: bytes) -> None:
|
||||
raise NotImplementedError()
|
||||
|
||||
def on_tick(self):
|
||||
self.__ticks_passed += 1
|
||||
|
||||
if self.__ticks_passed == 120:
|
||||
logging.debug("Peer failed to fetch metadata in time for info hash %s!", self.__info_hash.hex())
|
||||
self.when_error()
|
||||
|
||||
def on_receivable(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
@ -211,6 +220,8 @@ class DisposablePeer:
|
||||
except MemoryError:
|
||||
logging.exception("Could not allocate %.1f KiB for the metadata!", metadata_size / 1024)
|
||||
self.when_error()
|
||||
return
|
||||
|
||||
self.__metadata_size = metadata_size
|
||||
self.__ext_handshake_complete = True
|
||||
|
||||
|
@ -16,7 +16,6 @@ import array
|
||||
import collections
|
||||
import zlib
|
||||
import logging
|
||||
import random
|
||||
import socket
|
||||
import typing
|
||||
import os
|
||||
@ -54,7 +53,7 @@ class SybilNode:
|
||||
# stop; but until then, the total number of neighbours might exceed the threshold).
|
||||
self.__n_max_neighbours = 2000
|
||||
|
||||
logging.debug("SybilNode %s on %s initialized!", self.__true_id.hex().upper(), address)
|
||||
logging.info("SybilNode %s on %s initialized!", self.__true_id.hex().upper(), address)
|
||||
|
||||
@staticmethod
|
||||
def when_peer_found(info_hash: InfoHash, peer_addr: PeerAddress) -> None:
|
||||
@ -165,7 +164,7 @@ class SybilNode:
|
||||
return
|
||||
|
||||
# appendleft to prioritise GET_PEERS responses as they are the most fruitful ones!
|
||||
self.__outgoing_queue.append((addr, bencode.dumps({
|
||||
self.__outgoing_queue.appendleft((addr, bencode.dumps({
|
||||
b"y": b"r",
|
||||
b"t": transaction_id,
|
||||
b"r": {
|
||||
|
@ -119,7 +119,7 @@ class Database:
|
||||
self.__pending_files
|
||||
)
|
||||
cur.execute("COMMIT;")
|
||||
logging.debug("%d metadata (%d files) are committed to the database.",
|
||||
logging.info("%d metadata (%d files) are committed to the database.",
|
||||
len(self.__pending_metadata), len(self.__pending_files))
|
||||
self.__pending_metadata.clear()
|
||||
self.__pending_files.clear()
|
||||
|
@ -8,7 +8,7 @@ def read_file(path):
|
||||
|
||||
setup(
|
||||
name="magneticod",
|
||||
version="0.2.0",
|
||||
version="0.3.0",
|
||||
description="Autonomous BitTorrent DHT crawler and metadata fetcher.",
|
||||
long_description=read_file("README.rst"),
|
||||
url="http://magnetico.org",
|
||||
|
@ -14,6 +14,7 @@
|
||||
# <http://www.gnu.org/licenses/>.
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
@ -23,15 +24,22 @@ from magneticow import magneticow
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s")
|
||||
|
||||
arguments = parse_args()
|
||||
magneticow.app.arguments = arguments
|
||||
|
||||
http_server = gevent.wsgi.WSGIServer(("", arguments.port), magneticow.app)
|
||||
|
||||
magneticow.initialize_magneticod_db()
|
||||
|
||||
try:
|
||||
logging.info("magneticow is ready to serve!")
|
||||
http_server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
return 0
|
||||
finally:
|
||||
magneticow.close_db()
|
||||
|
||||
return 1
|
||||
|
||||
@ -71,4 +79,4 @@ def parse_args() -> dict:
|
||||
return parser.parse_args(sys.argv[1:])
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
sys.exit(main())
|
||||
|
@ -15,6 +15,7 @@
|
||||
import collections
|
||||
import functools
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
@ -31,6 +32,10 @@ Torrent = collections.namedtuple("torrent", ["info_hash", "name", "size", "disco
|
||||
app = flask.Flask(__name__)
|
||||
app.config.from_object(__name__)
|
||||
|
||||
# TODO: We should have been able to use flask.g but it does NOT persist across different requests so we resort back to
|
||||
# this. Investigate the cause and fix it (I suspect of Gevent).
|
||||
magneticod_db = None
|
||||
|
||||
|
||||
# Adapted from: http://flask.pocoo.org/snippets/8/
|
||||
# (c) Copyright 2010 - 2017 by Armin Ronacher
|
||||
@ -67,10 +72,8 @@ def requires_auth(f):
|
||||
@app.route("/")
|
||||
@requires_auth
|
||||
def home_page():
|
||||
magneticod_db = get_magneticod_db()
|
||||
|
||||
with magneticod_db:
|
||||
cur = magneticod_db.execute("SELECT count() FROM torrents ;")
|
||||
cur = magneticod_db.execute("SELECT MAX(ROWID) FROM torrents ;")
|
||||
n_torrents = cur.fetchone()[0]
|
||||
|
||||
return flask.render_template("homepage.html", n_torrents=n_torrents)
|
||||
@ -88,8 +91,6 @@ def torrents():
|
||||
|
||||
|
||||
def search_torrents():
|
||||
magneticod_db = get_magneticod_db()
|
||||
|
||||
search = flask.request.args["search"]
|
||||
page = int(flask.request.args.get("page", 0))
|
||||
|
||||
@ -128,8 +129,6 @@ def search_torrents():
|
||||
|
||||
|
||||
def newest_torrents():
|
||||
magneticod_db = get_magneticod_db()
|
||||
|
||||
page = int(flask.request.args.get("page", 0))
|
||||
|
||||
context = {
|
||||
@ -162,8 +161,6 @@ def newest_torrents():
|
||||
@app.route("/torrents/<info_hash>/", defaults={"name": None})
|
||||
@requires_auth
|
||||
def torrent_redirect(**kwargs):
|
||||
magnetico_db = get_magneticod_db()
|
||||
|
||||
try:
|
||||
info_hash = bytes.fromhex(kwargs["info_hash"])
|
||||
assert len(info_hash) == 20
|
||||
@ -183,7 +180,6 @@ def torrent_redirect(**kwargs):
|
||||
@app.route("/torrents/<info_hash>/<name>")
|
||||
@requires_auth
|
||||
def torrent(**kwargs):
|
||||
magneticod_db = get_magneticod_db()
|
||||
context = {}
|
||||
|
||||
try:
|
||||
@ -210,13 +206,13 @@ def torrent(**kwargs):
|
||||
return flask.render_template("torrent.html", **context)
|
||||
|
||||
|
||||
def get_magneticod_db():
|
||||
""" Opens a new database connection if there is none yet for the current application context. """
|
||||
if hasattr(flask.g, "magneticod_db"):
|
||||
return flask.g.magneticod_db
|
||||
def initialize_magneticod_db() -> None:
|
||||
global magneticod_db
|
||||
|
||||
logging.info("Connecting to magneticod's database...")
|
||||
|
||||
magneticod_db_path = os.path.join(appdirs.user_data_dir("magneticod"), "database.sqlite3")
|
||||
magneticod_db = flask.g.magneticod_db = sqlite3.connect(magneticod_db_path, isolation_level=None)
|
||||
magneticod_db = sqlite3.connect(magneticod_db_path, isolation_level=None)
|
||||
|
||||
with magneticod_db:
|
||||
magneticod_db.execute("PRAGMA journal_mode=WAL;")
|
||||
@ -232,11 +228,8 @@ def get_magneticod_db():
|
||||
|
||||
magneticod_db.create_function("rank", 1, utils.rank)
|
||||
|
||||
return magneticod_db
|
||||
|
||||
|
||||
@app.teardown_appcontext
|
||||
def close_magneticod_db(error):
|
||||
""" Closes the database again at the end of the request. """
|
||||
if hasattr(flask.g, "magneticod_db"):
|
||||
flask.g.magneticod_db.close()
|
||||
def close_db() -> None:
|
||||
logging.info("Closing magneticod database...")
|
||||
if magneticod_db is not None:
|
||||
magneticod_db.close()
|
||||
|
@ -17,7 +17,7 @@
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
Now with {{n_torrents}} torrents available !
|
||||
Now with {{ "{:,}".format(n_torrents) }} torrents available !
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
|
@ -8,7 +8,7 @@ def read_file(path):
|
||||
|
||||
setup(
|
||||
name="magneticow",
|
||||
version="0.2.0",
|
||||
version="0.3.0",
|
||||
description="Lightweight web interface for magnetico.",
|
||||
long_description=read_file("README.rst"),
|
||||
url="http://magnetico.org",
|
||||
|
Loading…
Reference in New Issue
Block a user