From ae691ada79cd9c6a607fcb08ba902b22b5cf1755 Mon Sep 17 00:00:00 2001 From: Bora Alper Date: Thu, 2 Nov 2017 23:15:13 +0000 Subject: [PATCH] started cleaning up for v0.7.0 I've decided instead to release a minimum viable product for v0.7.0 and get some feedback from the community, and most importantly some motivation as well to be able to keep working on magnetico as it currently feels like a Sisyphean where the development seem to never going to end... --- .gitignore | 219 +++------ src/magneticod/Gopkg.toml => Gopkg.toml | 22 +- Makefile | 9 + bin/.gitignore | 4 - magneticod/bittorrent/operations.go | 463 ++++++++++++++++++ magneticod/bittorrent/sinkMetadata.go | 103 ++++ .../dht/mainline/codec.go | 35 +- .../dht/mainline/codec_test.go | 51 +- .../dht/mainline/protocol.go | 54 +- .../dht/mainline/protocol_test.go | 51 +- .../dht/mainline/service.go | 35 +- .../dht/mainline/transport.go | 13 +- .../dht/mainline/transport_test.go | 4 +- .../magneticod => magneticod}/dht/managers.go | 9 +- magneticod/main.go | 157 ++++++ {src/magneticod => magneticod}/main_test.go | 1 - magneticow/.dockerignore | 2 - magneticow/Dockerfile | 10 - magneticow/MANIFEST.in | 2 - magneticow/README.rst | 205 -------- .../data/static/assets/feed.png | Bin .../data/static/assets/magnet.gif | Bin .../static/fonts/NotoMono/LICENSE_OFL.txt | 0 .../data/static/fonts/NotoMono/Regular.ttf | Bin .../data/static/fonts/NotoSansUI/Bold.ttf | Bin .../static/fonts/NotoSansUI/BoldItalic.ttf | Bin .../data/static/fonts/NotoSansUI/Italic.ttf | Bin .../static/fonts/NotoSansUI/LICENSE_OFL.txt | 0 .../data/static/fonts/NotoSansUI/Regular.ttf | Bin .../data/static/scripts/plotly-v1.26.1.min.js | 0 .../data/static/scripts/statistics.js | 0 .../data/static/scripts/torrent.js | 0 .../data/static/styles/essential.css | 8 +- magneticow/data/static/styles/homepage.css | 27 + .../data/static/styles/reset.css | 0 .../data/static/styles/statistics.css | 0 .../data/static/styles/torrent.css | 0 .../data/static/styles/torrents.css | 0 magneticow/data/templates/feed.xml | 13 + .../data/templates/homepage.html | 5 +- .../data/templates/statistics.html | 10 +- .../data/templates/torrent.html | 10 +- magneticow/data/templates/torrents.html | 90 ++++ magneticow/docs/API/README.md | 33 -- magneticow/docs/API/v0.md | 93 ---- magneticow/magneticow/__init__.py | 14 - magneticow/magneticow/__main__.py | 93 ---- magneticow/magneticow/authorization.py | 61 --- magneticow/magneticow/magneticow.py | 295 ----------- magneticow/magneticow/utils.py | 69 --- magneticow/main.go | 304 ++++++++++++ magneticow/setup.py | 41 -- magneticow/systemd/magneticow.service | 10 - persistence/interface.go | 117 +++++ {src/persistence => persistence}/sqlite3.go | 243 ++++----- pkg/.gitignore | 4 - pylintrc | 407 --------------- src/magneticod/bittorrent/operations.go | 185 ------- src/magneticod/bittorrent/sinkFile.go | 117 ----- src/magneticod/bittorrent/sinkMetadata.go | 89 ---- src/magneticod/coordinators.go | 111 ----- src/magneticod/dht/mainline/bloomFilter.go | 59 --- .../dht/mainline/bloomFilter_test.go | 64 --- src/magneticod/main.go | 284 ----------- src/magneticod/persistence/database.go | 385 --------------- src/magneticod/persistence/database_test.go | 20 - src/magneticow/Gopkg.toml | 26 - src/magneticow/bindata.go | 262 ---------- .../data/static/styles/homepage.css | 38 -- src/magneticow/data/templates/feed.xml | 16 - src/magneticow/data/templates/torrents.html | 66 --- src/magneticow/main.go | 108 ---- src/persistence/Gopkg.toml | 26 - src/persistence/interface.go | 116 ----- src/persistence/mysql.go | 1 - src/persistence/postgresql.go | 1 - 76 files changed, 1568 insertions(+), 3802 deletions(-) rename src/magneticod/Gopkg.toml => Gopkg.toml (72%) create mode 100644 Makefile delete mode 100644 bin/.gitignore create mode 100644 magneticod/bittorrent/operations.go create mode 100644 magneticod/bittorrent/sinkMetadata.go rename {src/magneticod => magneticod}/dht/mainline/codec.go (98%) rename {src/magneticod => magneticod}/dht/mainline/codec_test.go (84%) rename {src/magneticod => magneticod}/dht/mainline/protocol.go (89%) rename {src/magneticod => magneticod}/dht/mainline/protocol_test.go (84%) rename {src/magneticod => magneticod}/dht/mainline/service.go (91%) rename {src/magneticod => magneticod}/dht/mainline/transport.go (96%) rename {src/magneticod => magneticod}/dht/mainline/transport_test.go (99%) rename {src/magneticod => magneticod}/dht/managers.go (91%) create mode 100644 magneticod/main.go rename {src/magneticod => magneticod}/main_test.go (99%) delete mode 100644 magneticow/.dockerignore delete mode 100644 magneticow/Dockerfile delete mode 100644 magneticow/MANIFEST.in delete mode 100644 magneticow/README.rst rename {src/magneticow => magneticow}/data/static/assets/feed.png (100%) rename {src/magneticow => magneticow}/data/static/assets/magnet.gif (100%) rename {src/magneticow => magneticow}/data/static/fonts/NotoMono/LICENSE_OFL.txt (100%) rename {src/magneticow => magneticow}/data/static/fonts/NotoMono/Regular.ttf (100%) rename {src/magneticow => magneticow}/data/static/fonts/NotoSansUI/Bold.ttf (100%) rename {src/magneticow => magneticow}/data/static/fonts/NotoSansUI/BoldItalic.ttf (100%) rename {src/magneticow => magneticow}/data/static/fonts/NotoSansUI/Italic.ttf (100%) rename {src/magneticow => magneticow}/data/static/fonts/NotoSansUI/LICENSE_OFL.txt (100%) rename {src/magneticow => magneticow}/data/static/fonts/NotoSansUI/Regular.ttf (100%) rename {src/magneticow => magneticow}/data/static/scripts/plotly-v1.26.1.min.js (100%) rename {src/magneticow => magneticow}/data/static/scripts/statistics.js (100%) rename {src/magneticow => magneticow}/data/static/scripts/torrent.js (100%) rename {src/magneticow => magneticow}/data/static/styles/essential.css (94%) create mode 100644 magneticow/data/static/styles/homepage.css rename {src/magneticow => magneticow}/data/static/styles/reset.css (100%) rename {src/magneticow => magneticow}/data/static/styles/statistics.css (100%) rename {src/magneticow => magneticow}/data/static/styles/torrent.css (100%) rename {src/magneticow => magneticow}/data/static/styles/torrents.css (100%) create mode 100644 magneticow/data/templates/feed.xml rename {src/magneticow => magneticow}/data/templates/homepage.html (70%) rename {src/magneticow => magneticow}/data/templates/statistics.html (50%) rename {src/magneticow => magneticow}/data/templates/torrent.html (75%) create mode 100644 magneticow/data/templates/torrents.html delete mode 100644 magneticow/docs/API/README.md delete mode 100644 magneticow/docs/API/v0.md delete mode 100644 magneticow/magneticow/__init__.py delete mode 100644 magneticow/magneticow/__main__.py delete mode 100644 magneticow/magneticow/authorization.py delete mode 100644 magneticow/magneticow/magneticow.py delete mode 100644 magneticow/magneticow/utils.py create mode 100644 magneticow/main.go delete mode 100644 magneticow/setup.py delete mode 100644 magneticow/systemd/magneticow.service create mode 100644 persistence/interface.go rename {src/persistence => persistence}/sqlite3.go (64%) delete mode 100644 pkg/.gitignore delete mode 100644 pylintrc delete mode 100644 src/magneticod/bittorrent/operations.go delete mode 100644 src/magneticod/bittorrent/sinkFile.go delete mode 100644 src/magneticod/bittorrent/sinkMetadata.go delete mode 100644 src/magneticod/coordinators.go delete mode 100644 src/magneticod/dht/mainline/bloomFilter.go delete mode 100644 src/magneticod/dht/mainline/bloomFilter_test.go delete mode 100644 src/magneticod/main.go delete mode 100644 src/magneticod/persistence/database.go delete mode 100644 src/magneticod/persistence/database_test.go delete mode 100644 src/magneticow/Gopkg.toml delete mode 100644 src/magneticow/bindata.go delete mode 100644 src/magneticow/data/static/styles/homepage.css delete mode 100644 src/magneticow/data/templates/feed.xml delete mode 100644 src/magneticow/data/templates/torrents.html delete mode 100644 src/magneticow/main.go delete mode 100644 src/persistence/Gopkg.toml delete mode 100644 src/persistence/interface.go delete mode 100644 src/persistence/mysql.go delete mode 100644 src/persistence/postgresql.go diff --git a/.gitignore b/.gitignore index 0808890..0c20642 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,26 @@ -src/magneticod/vendor -src/magneticod/Gopkg.lock -src/magneticow/vendor -src/magneticow/Gopkg.lock -src/persistence/vendor -src/persistence/Gopkg.lock +vendor/ +.idea/ +Gopkg.lock +magneticow/bindata.go -# Created by https://www.gitignore.io/api/linux,python,pycharm + +# Created by https://www.gitignore.io/api/go,linux,macos,windows + +### Go ### +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ ### Linux ### *~ @@ -22,164 +37,52 @@ src/persistence/Gopkg.lock # .nfs files are created when an open file is removed but is still being accessed .nfs* -### PyCharm ### -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 +### macOS ### +*.DS_Store +.AppleDouble +.LSOverride -.idea/ +# Icon must end with two \r +Icon -# User-specific stuff: -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/dictionaries +# Thumbnails +._* -# Sensitive or high-churn files: -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.xml -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent -# Gradle: -.idea/**/gradle.xml -.idea/**/libraries +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk -# Mongo Explorer plugin: -.idea/**/mongoSettings.xml +### Windows ### +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db -## File-based project format: -*.iws +# Folder config file +Desktop.ini -## Plugin-specific files: +# Recycle Bin used on file shares +$RECYCLE.BIN/ -# IntelliJ -/out/ +# Windows Installer files +*.cab +*.msi +*.msm +*.msp -# mpeltonen/sbt-idea plugin -.idea_modules/ +# Windows shortcuts +*.lnk -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties - -### PyCharm Patch ### -# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 - -# *.iml -# modules.xml -# .idea/misc.xml -# *.ipr - -# Sonarlint plugin -.idea/sonarlint - -### Python ### -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject - -# Rope project settings -.ropeproject - -# End of https://www.gitignore.io/api/linux,python,pycharm - -docker-compose.override.yml -.mypy_cache +# End of https://www.gitignore.io/api/go,linux,macos,windows diff --git a/src/magneticod/Gopkg.toml b/Gopkg.toml similarity index 72% rename from src/magneticod/Gopkg.toml rename to Gopkg.toml index 27101f2..d36adfa 100644 --- a/src/magneticod/Gopkg.toml +++ b/Gopkg.toml @@ -21,6 +21,14 @@ # version = "2.4.0" +[[constraint]] + branch = "master" + name = "github.com/Wessie/appdirs" + +[[constraint]] + branch = "master" + name = "github.com/anacrolix/dht" + [[constraint]] branch = "master" name = "github.com/anacrolix/missinggo" @@ -31,7 +39,11 @@ [[constraint]] branch = "master" - name = "github.com/bradfitz/iter" + name = "github.com/dustin/go-humanize" + +[[constraint]] + name = "github.com/gorilla/mux" + version = "1.5.0" [[constraint]] name = "github.com/jessevdk/go-flags" @@ -39,8 +51,12 @@ [[constraint]] name = "github.com/mattn/go-sqlite3" - version = "1.2.0" + version = "1.3.0" + +[[constraint]] + name = "github.com/willf/bloom" + version = "2.0.3" [[constraint]] name = "go.uber.org/zap" - version = "1.5.0" + version = "1.7.1" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..eaf487e --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +all: magneticod magneticow + +magneticod: + go install magnetico/magneticod + +magneticow: + # TODO: minify files! + go-bindata -o="magneticow/bindata.go" -prefix="magneticow/data/" magneticow/data/... + go install magnetico/magneticow diff --git a/bin/.gitignore b/bin/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/bin/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/magneticod/bittorrent/operations.go b/magneticod/bittorrent/operations.go new file mode 100644 index 0000000..7b3352a --- /dev/null +++ b/magneticod/bittorrent/operations.go @@ -0,0 +1,463 @@ +package bittorrent + +import ( + "bytes" + "crypto/sha1" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "net" + "time" + + "github.com/anacrolix/torrent/bencode" + "github.com/anacrolix/torrent/metainfo" + + "go.uber.org/zap" + + "magnetico/persistence" +) + +const MAX_METADATA_SIZE = 10 * 1024 * 1024 + +type rootDict struct { + M mDict `bencode:"m"` + MetadataSize int `bencode:"metadata_size"` +} + +type mDict struct { + UTMetadata int `bencode:"ut_metadata"` +} + +type extDict struct { + MsgType int `bencode:"msg_type"` + Piece int `bencode:"piece"` +} + +func (ms *MetadataSink) awaitMetadata(infoHash metainfo.Hash, peer Peer) { + conn, err := net.DialTCP("tcp", nil, peer.Addr) + if err != nil { + zap.L().Debug( + "awaitMetadata couldn't connect to the peer!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", peer.Addr.String()), + zap.Error(err), + ) + return + } + defer conn.Close() + + err = conn.SetNoDelay(true) + if err != nil { + zap.L().Panic( + "Couldn't set NODELAY!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", peer.Addr.String()), + zap.Error(err), + ) + return + } + err = conn.SetDeadline(time.Now().Add(ms.deadline)) + if err != nil { + zap.L().Panic( + "Couldn't set the deadline!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", peer.Addr.String()), + zap.Error(err), + ) + return + } + + // State Variables + var isExtHandshakeDone, done bool + var ut_metadata, metadataReceived, metadataSize int + var metadata []byte + + lHandshake := []byte(fmt.Sprintf( + "\x13BitTorrent protocol\x00\x00\x00\x00\x00\x10\x00\x01%s%s", + infoHash[:], + ms.clientID, + )) + if len(lHandshake) != 68 { + zap.L().Panic( + "Generated BitTorrent handshake is not of length 68!", + zap.ByteString("infoHash", infoHash[:]), + zap.Int("len_lHandshake", len(lHandshake)), + ) + } + err = writeAll(conn, lHandshake) + if err != nil { + zap.L().Debug( + "Couldn't write BitTorrent handshake!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", peer.Addr.String()), + zap.Error(err), + ) + return + } + + zap.L().Debug("BitTorrent handshake sent, waiting for the remote's...") + + rHandshake, err := readExactly(conn, 68) + if err != nil { + zap.L().Debug( + "Couldn't read remote BitTorrent handshake!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", peer.Addr.String()), + zap.Error(err), + ) + return + } + if !bytes.HasPrefix(rHandshake, []byte("\x13BitTorrent protocol")) { + zap.L().Debug( + "Remote BitTorrent handshake is not what it is supposed to be!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", peer.Addr.String()), + zap.ByteString("rHandshake[:20]", rHandshake[:20]), + ) + return + } + + // __on_bt_handshake + // ================ + if rHandshake[25] != 16 { // TODO (later): do *not* compare the whole byte, check the bit instead! (0x10) + zap.L().Debug( + "Peer does not support the extension protocol!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", peer.Addr.String()), + ) + return + } + + writeAll(conn, []byte("\x00\x00\x00\x1a\x14\x00d1:md11:ut_metadatai1eee")) + zap.L().Debug( + "Extension handshake sent, waiting for the remote's...", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", peer.Addr.String()), + ) + + // the loop! + // ========= + for !done { + rLengthB, err := readExactly(conn, 4) + if err != nil { + zap.L().Debug( + "Couldn't read the first 4 bytes from the remote peer in the loop!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", peer.Addr.String()), + zap.Error(err), + ) + return + } + rLength := bigEndianToInt(rLengthB) + rMessage, err := readExactly(conn, rLength) + if err != nil { + zap.L().Debug( + "Couldn't read the rest of the message from the remote peer in the loop!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", peer.Addr.String()), + zap.Error(err), + ) + return + } + + // __on_message + // ------------ + if rMessage[0] != 0x14 { // We are interested only in extension messages, whose first byte is always 0x14 + zap.L().Debug( + "Ignoring the non-extension message.", + zap.ByteString("infoHash", infoHash[:]), + ) + continue + } + + if rMessage[1] == 0x00 { // Extension Handshake has the Extension Message ID = 0x00 + // __on_ext_handshake_message(message[2:]) + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + // TODO: continue editing log messages from here + + if isExtHandshakeDone { + return + } + + rRootDict := new(rootDict) + err := bencode.Unmarshal(rMessage[2:], rRootDict) + if err != nil { + zap.L().Debug("Couldn't unmarshal extension handshake!", zap.Error(err)) + return + } + + if rRootDict.MetadataSize <= 0 || rRootDict.MetadataSize > MAX_METADATA_SIZE { + zap.L().Debug("Unacceptable metadata size!", zap.Int("metadata_size", rRootDict.MetadataSize)) + return + } + + ut_metadata = rRootDict.M.UTMetadata // Save the ut_metadata code the remote peer uses + metadataSize = rRootDict.MetadataSize + metadata = make([]byte, metadataSize) + isExtHandshakeDone = true + + zap.L().Debug("GOT EXTENSION HANDSHAKE!", zap.Int("ut_metadata", ut_metadata), zap.Int("metadata_size", metadataSize)) + + // Request all the pieces of metadata + n_pieces := int(math.Ceil(float64(metadataSize) / math.Pow(2, 14))) + for piece := 0; piece < n_pieces; piece++ { + // __request_metadata_piece(piece) + // ............................... + extDictDump, err := bencode.Marshal(extDict{ + MsgType: 0, + Piece: piece, + }) + if err != nil { + zap.L().Warn("Couldn't marshal extDictDump!", zap.Error(err)) + return + } + writeAll(conn, []byte(fmt.Sprintf( + "%s\x14%s%s", + intToBigEndian(2+len(extDictDump), 4), + intToBigEndian(ut_metadata, 1), + extDictDump, + ))) + } + + zap.L().Warn("requested all metadata pieces!") + + } else if rMessage[1] == 0x01 { + // __on_ext_message(message[2:]) + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + rMessageBuf := bytes.NewBuffer(rMessage[2:]) + rExtDict := new(extDict) + // TODO: We monkey-patched anacrolix/torrent! + err := bencode.NewDecoder2(rMessageBuf).Decode(rExtDict) + if err != nil { + zap.L().Warn("Couldn't decode extension message in the loop!", zap.Error(err)) + return + } + + if rExtDict.MsgType == 1 { // data + // Get the unread bytes! + metadataPiece := rMessageBuf.Bytes() + piece := rExtDict.Piece + // metadata[piece * 2**14: piece * 2**14 + len(metadataPiece)] = metadataPiece is how it'd be done in Python + copy(metadata[piece*int(math.Pow(2, 14)):piece*int(math.Pow(2, 14))+len(metadataPiece)], metadataPiece) + metadataReceived += len(metadataPiece) + done = metadataReceived == metadataSize + + // BEP 9 explicitly states: + // > If the piece is the last piece of the metadata, it may be less than 16kiB. If + // > it is not the last piece of the metadata, it MUST be 16kiB. + // + // Hence... + // ... if the length of @metadataPiece is more than 16kiB, we err. + if len(metadataPiece) > 16*1024 { + zap.L().Debug( + "metadataPiece is bigger than 16kiB!", + zap.Int("len_metadataPiece", len(metadataPiece)), + zap.Int("metadataReceived", metadataReceived), + zap.Int("metadataSize", metadataSize), + zap.Int("metadataPieceIndex", bytes.Index(rMessage, metadataPiece)), + ) + return + } + + // ... if the length of @metadataPiece is less than 16kiB AND metadata is NOT + // complete (!done) then we err. + if len(metadataPiece) < 16*1024 && !done { + zap.L().Debug( + "metadataPiece is less than 16kiB and metadata is incomplete!", + zap.Int("len_metadataPiece", len(metadataPiece)), + zap.Int("metadataReceived", metadataReceived), + zap.Int("metadataSize", metadataSize), + zap.Int("metadataPieceIndex", bytes.Index(rMessage, metadataPiece)), + ) + return + } + + if metadataReceived > metadataSize { + zap.L().Debug( + "metadataReceived is greater than metadataSize!", + zap.Int("len_metadataPiece", len(metadataPiece)), + zap.Int("metadataReceived", metadataReceived), + zap.Int("metadataSize", metadataSize), + zap.Int("metadataPieceIndex", bytes.Index(rMessage, metadataPiece)), + ) + return + } + + zap.L().Debug( + "Fetching...", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", conn.RemoteAddr().String()), + zap.Int("metadataReceived", metadataReceived), + zap.Int("metadataSize", metadataSize), + ) + } else if rExtDict.MsgType == 2 { // reject + zap.L().Debug( + "Remote peer rejected sending metadata!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("remotePeerAddr", conn.RemoteAddr().String()), + ) + return + } + + } else { + zap.L().Debug( + "Message is not an ut_metadata message! (ignoring)", + zap.ByteString("msg", rMessage[:100]), + ) + // no return! + } + } + + zap.L().Debug( + "Metadata is complete, verifying the checksum...", + zap.ByteString("infoHash", infoHash[:]), + ) + + sha1Sum := sha1.Sum(metadata) + if !bytes.Equal(sha1Sum[:], infoHash[:]) { + zap.L().Debug( + "Info-hash mismatch!", + zap.ByteString("expectedInfoHash", infoHash[:]), + zap.ByteString("actualInfoHash", sha1Sum[:]), + ) + return + } + + zap.L().Debug( + "Checksum verified, checking the info dictionary...", + zap.ByteString("infoHash", infoHash[:]), + ) + + info := new(metainfo.Info) + err = bencode.Unmarshal(metadata, info) + if err != nil { + zap.L().Debug( + "Couldn't unmarshal info bytes!", + zap.ByteString("infoHash", infoHash[:]), + zap.Error(err), + ) + return + } + err = validateInfo(info) + if err != nil { + zap.L().Debug( + "Bad info dictionary!", + zap.ByteString("infoHash", infoHash[:]), + zap.Error(err), + ) + return + } + + var files []persistence.File + for _, file := range info.Files { + if file.Length < 0 { + zap.L().Debug( + "File size is less than zero!", + zap.ByteString("infoHash", infoHash[:]), + zap.String("filePath", file.DisplayPath(info)), + zap.Int64("fileSize", file.Length), + ) + return + } + + files = append(files, persistence.File{ + Size: file.Length, + Path: file.DisplayPath(info), + }) + } + + var totalSize uint64 + for _, file := range files { + totalSize += uint64(file.Size) + } + + zap.L().Debug( + "Flushing metadata...", + zap.ByteString("infoHash", infoHash[:]), + ) + + ms.flush(Metadata{ + InfoHash: infoHash[:], + Name: info.Name, + TotalSize: totalSize, + DiscoveredOn: time.Now().Unix(), + Files: files, + }) +} + +// COPIED FROM anacrolix/torrent +func validateInfo(info *metainfo.Info) error { + if len(info.Pieces)%20 != 0 { + return errors.New("pieces has invalid length") + } + if info.PieceLength == 0 { + if info.TotalLength() != 0 { + return errors.New("zero piece length") + } + } else { + if int((info.TotalLength()+info.PieceLength-1)/info.PieceLength) != info.NumPieces() { + return errors.New("piece count and file lengths are at odds") + } + } + return nil +} + +func writeAll(c *net.TCPConn, b []byte) error { + for len(b) != 0 { + n, err := c.Write(b) + if err != nil { + return err + } + b = b[n:] + } + return nil +} + +func readExactly(c *net.TCPConn, n int) ([]byte, error) { + b := make([]byte, n) + _, err := io.ReadFull(c, b) + return b, err +} + +// TODO: add bounds checking! +func intToBigEndian(i int, n int) []byte { + b := make([]byte, n) + switch n { + case 1: + b = []byte{byte(i)} + + case 2: + binary.BigEndian.PutUint16(b, uint16(i)) + + case 4: + binary.BigEndian.PutUint32(b, uint32(i)) + + default: + panic(fmt.Sprintf("n must be 1, 2, or 4!")) + } + + if len(b) != n { + panic(fmt.Sprintf("postcondition failed: len(b) != n in intToBigEndian (i %d, n %d, len b %d, b %s)", i, n, len(b), b)) + } + + return b +} + +func bigEndianToInt(b []byte) int { + switch len(b) { + case 1: + return int(b[0]) + + case 2: + return int(binary.BigEndian.Uint16(b)) + + case 4: + return int(binary.BigEndian.Uint32(b)) + + default: + panic(fmt.Sprintf("bigEndianToInt: b is too long! (%d bytes)", len(b))) + } +} diff --git a/magneticod/bittorrent/sinkMetadata.go b/magneticod/bittorrent/sinkMetadata.go new file mode 100644 index 0000000..55842f3 --- /dev/null +++ b/magneticod/bittorrent/sinkMetadata.go @@ -0,0 +1,103 @@ +package bittorrent + +import ( + "crypto/rand" + "fmt" + "net" + "strings" + "time" + + "go.uber.org/zap" + + "magnetico/magneticod/dht/mainline" + "magnetico/persistence" +) + +type Metadata struct { + InfoHash []byte + // Name should be thought of "Title" of the torrent. For single-file torrents, it is the name + // of the file, and for multi-file torrents, it is the name of the root directory. + Name string + TotalSize uint64 + DiscoveredOn int64 + // Files must be populated for both single-file and multi-file torrents! + Files []persistence.File +} + +type Peer struct { + Addr *net.TCPAddr +} + +type MetadataSink struct { + clientID []byte + deadline time.Duration + drain chan Metadata + terminated bool + termination chan interface{} +} + +func NewMetadataSink(deadline time.Duration) *MetadataSink { + ms := new(MetadataSink) + + ms.clientID = make([]byte, 20) + _, err := rand.Read(ms.clientID) + if err != nil { + zap.L().Panic("sinkMetadata couldn't read 20 random bytes for client ID!", zap.Error(err)) + } + // TODO: remove this + if len(ms.clientID) != 20 { + panic("CLIENT ID NOT 20!") + } + + ms.deadline = deadline + ms.drain = make(chan Metadata) + ms.termination = make(chan interface{}) + return ms +} + +func (ms *MetadataSink) Sink(res mainline.TrawlingResult) { + if ms.terminated { + zap.L().Panic("Trying to Sink() an already closed MetadataSink!") + } + + IPs := res.PeerIP.String() + var rhostport string + if IPs == "" { + zap.L().Debug("MetadataSink.Sink: Peer IP is nil!") + return + } else if IPs[0] == '?' { + zap.L().Debug("MetadataSink.Sink: Peer IP is invalid!") + return + } else if strings.ContainsRune(IPs, ':') { // IPv6 + rhostport = fmt.Sprintf("[%s]:%d", IPs, res.PeerPort) + } else { // IPv4 + rhostport = fmt.Sprintf("%s:%d", IPs, res.PeerPort) + } + + raddr, err := net.ResolveTCPAddr("tcp", rhostport) + if err != nil { + zap.L().Debug("MetadataSink.Sink: Couldn't resolve peer address!", zap.Error(err)) + return + } + + go ms.awaitMetadata(res.InfoHash, Peer{Addr: raddr}) +} + +func (ms *MetadataSink) Drain() <-chan Metadata { + if ms.terminated { + zap.L().Panic("Trying to Drain() an already closed MetadataSink!") + } + return ms.drain +} + +func (ms *MetadataSink) Terminate() { + ms.terminated = true + close(ms.termination) + close(ms.drain) +} + +func (ms *MetadataSink) flush(result Metadata) { + if !ms.terminated { + ms.drain <- result + } +} diff --git a/src/magneticod/dht/mainline/codec.go b/magneticod/dht/mainline/codec.go similarity index 98% rename from src/magneticod/dht/mainline/codec.go rename to magneticod/dht/mainline/codec.go index bd559c5..810a25b 100644 --- a/src/magneticod/dht/mainline/codec.go +++ b/magneticod/dht/mainline/codec.go @@ -1,4 +1,3 @@ - // TODO: This file, as a whole, needs a little skim-through to clear things up, sprinkle a little // documentation here and there, and also to make the test coverage 100%. // It, most importantly, lacks IPv6 support, if it's not altogether messy and unreliable @@ -7,17 +6,16 @@ package mainline import ( - "fmt" "encoding/binary" + "fmt" "net" - "github.com/anacrolix/torrent/bencode" "github.com/anacrolix/missinggo/iter" - "regexp" + "github.com/anacrolix/torrent/bencode" "github.com/willf/bloom" + "regexp" ) - type Message struct { // Query method (one of 4: "ping", "find_node", "get_peers", "announce_peer") Q string `bencode:"q,omitempty"` @@ -33,7 +31,6 @@ type Message struct { E Error `bencode:"e,omitempty"` } - type QueryArguments struct { // ID of the quirying Node ID []byte `bencode:"id"` @@ -64,7 +61,6 @@ type QueryArguments struct { Scrape int `bencode:"noseed,omitempty"` } - type ResponseValues struct { // ID of the querying node ID []byte `bencode:"id"` @@ -85,32 +81,26 @@ type ResponseValues struct { // TODO: write marshallers for those fields above ^^ } - type Error struct { - Code int + Code int Message []byte } - // Represents peer address in either IPv6 or IPv4 form. type CompactPeer struct { IP net.IP Port int } - type CompactPeers []CompactPeer - type CompactNodeInfo struct { ID []byte Addr net.UDPAddr } - type CompactNodeInfos []CompactNodeInfo - // This allows bencode.Unmarshal to do better than a string or []byte. func (cps *CompactPeers) UnmarshalBencode(b []byte) (err error) { var bb []byte @@ -131,7 +121,6 @@ func (cps CompactPeers) MarshalBinary() (ret []byte, err error) { return } - func (cp CompactPeer) MarshalBencode() (ret []byte, err error) { ip := cp.IP if ip4 := ip.To4(); ip4 != nil { @@ -143,7 +132,6 @@ func (cp CompactPeer) MarshalBencode() (ret []byte, err error) { return bencode.Marshal(ret) } - func (cp *CompactPeer) UnmarshalBinary(b []byte) error { switch len(b) { case 18: @@ -159,7 +147,6 @@ func (cp *CompactPeer) UnmarshalBinary(b []byte) error { return nil } - func (cp *CompactPeer) UnmarshalBencode(b []byte) (err error) { var _b []byte err = bencode.Unmarshal(b, &_b) @@ -169,7 +156,6 @@ func (cp *CompactPeer) UnmarshalBencode(b []byte) (err error) { return cp.UnmarshalBinary(_b) } - func UnmarshalCompactPeers(b []byte) (ret []CompactPeer, err error) { num := len(b) / 6 ret = make([]CompactPeer, num) @@ -183,7 +169,6 @@ func UnmarshalCompactPeers(b []byte) (ret []CompactPeer, err error) { return } - // This allows bencode.Unmarshal to do better than a string or []byte. func (cnis *CompactNodeInfos) UnmarshalBencode(b []byte) (err error) { var bb []byte @@ -195,9 +180,8 @@ func (cnis *CompactNodeInfos) UnmarshalBencode(b []byte) (err error) { return } - func UnmarshalCompactNodeInfos(b []byte) (ret []CompactNodeInfo, err error) { - if len(b) % 26 != 0 { + if len(b)%26 != 0 { err = fmt.Errorf("compact node is not a multiple of 26") return } @@ -215,7 +199,6 @@ func UnmarshalCompactNodeInfos(b []byte) (ret []CompactNodeInfo, err error) { return } - func (cni *CompactNodeInfo) UnmarshalBinary(b []byte) error { copy(cni.ID[:], b) b = b[len(cni.ID):] @@ -227,10 +210,13 @@ func (cni *CompactNodeInfo) UnmarshalBinary(b []byte) error { return nil } - func (cnis CompactNodeInfos) MarshalBencode() ([]byte, error) { var ret []byte + if len(cnis) == 0 { + return []byte("0:"), nil + } + for _, cni := range cnis { ret = append(ret, cni.MarshalBinary()...) } @@ -238,7 +224,6 @@ func (cnis CompactNodeInfos) MarshalBencode() ([]byte, error) { return bencode.Marshal(ret) } - func (cni CompactNodeInfo) MarshalBinary() []byte { ret := make([]byte, 20) @@ -252,12 +237,10 @@ func (cni CompactNodeInfo) MarshalBinary() []byte { return ret } - func (e Error) MarshalBencode() ([]byte, error) { return []byte(fmt.Sprintf("li%de%d:%se", e.Code, len(e.Message), e.Message)), nil } - func (e *Error) UnmarshalBencode(b []byte) (err error) { var code, msgLen int diff --git a/src/magneticod/dht/mainline/codec_test.go b/magneticod/dht/mainline/codec_test.go similarity index 84% rename from src/magneticod/dht/mainline/codec_test.go rename to magneticod/dht/mainline/codec_test.go index afc385f..88a5b5f 100644 --- a/src/magneticod/dht/mainline/codec_test.go +++ b/magneticod/dht/mainline/codec_test.go @@ -9,8 +9,7 @@ import ( "github.com/anacrolix/torrent/bencode" ) - -var codecTest_validInstances = []struct{ +var codecTest_validInstances = []struct { data []byte msg Message }{ @@ -46,7 +45,7 @@ var codecTest_validInstances = []struct{ Y: "q", Q: "find_node", A: QueryArguments{ - ID: []byte("abcdefghij0123456789"), + ID: []byte("abcdefghij0123456789"), Target: []byte("mnopqrstuvwxyz123456"), }, }, @@ -58,7 +57,7 @@ var codecTest_validInstances = []struct{ T: []byte("aa"), Y: "r", R: ResponseValues{ - ID: []byte("0123456789abcdefghij"), + ID: []byte("0123456789abcdefghij"), Nodes: []CompactNodeInfo{}, }, }, @@ -73,7 +72,7 @@ var codecTest_validInstances = []struct{ ID: []byte("0123456789abcdefghij"), Nodes: []CompactNodeInfo{ { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, }, @@ -90,35 +89,35 @@ var codecTest_validInstances = []struct{ ID: []byte("0123456789abcdefghij"), Nodes: []CompactNodeInfo{ { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("zyxwvutsrqponmlkjihg"), + ID: []byte("zyxwvutsrqponmlkjihg"), Addr: net.UDPAddr{IP: []byte("\xf5\x8e\x82\x8b"), Port: 6931, Zone: ""}, }, }, @@ -133,7 +132,7 @@ var codecTest_validInstances = []struct{ Y: "q", Q: "get_peers", A: QueryArguments{ - ID: []byte("abcdefghij0123456789"), + ID: []byte("abcdefghij0123456789"), InfoHash: []byte("mnopqrstuvwxyz123456"), }, }, @@ -145,7 +144,7 @@ var codecTest_validInstances = []struct{ T: []byte("aa"), Y: "r", R: ResponseValues{ - ID: []byte("abcdefghij0123456789"), + ID: []byte("abcdefghij0123456789"), Token: []byte("aoeusnth"), Values: []CompactPeer{ {IP: []byte("axje"), Port: 11893}, @@ -161,15 +160,15 @@ var codecTest_validInstances = []struct{ T: []byte("aa"), Y: "r", R: ResponseValues{ - ID: []byte("abcdefghij0123456789"), + ID: []byte("abcdefghij0123456789"), Token: []byte("aoeusnth"), Nodes: []CompactNodeInfo{ { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("zyxwvutsrqponmlkjihg"), + ID: []byte("zyxwvutsrqponmlkjihg"), Addr: net.UDPAddr{IP: []byte("\xf5\x8e\x82\x8b"), Port: 6931, Zone: ""}, }, }, @@ -184,10 +183,10 @@ var codecTest_validInstances = []struct{ Y: "q", Q: "announce_peer", A: QueryArguments{ - ID: []byte("abcdefghij0123456789"), + ID: []byte("abcdefghij0123456789"), InfoHash: []byte("mnopqrstuvwxyz123456"), - Port: 6881, - Token: []byte("aoeusnth"), + Port: 6881, + Token: []byte("aoeusnth"), }, }, }, @@ -203,32 +202,30 @@ var codecTest_validInstances = []struct{ // TODO: Add announce_peer Query with optional `implied_port` argument. } - func TestUnmarshal(t *testing.T) { for i, instance := range codecTest_validInstances { msg := Message{} err := bencode.Unmarshal(instance.data, &msg) if err != nil { - t.Errorf("Error while unmarshalling valid data #%d: %v", i + 1, err) + t.Errorf("Error while unmarshalling valid data #%d: %v", i+1, err) continue } if reflect.DeepEqual(msg, instance.msg) != true { t.Errorf("Valid data #%d unmarshalled wrong!\n\tGot : %+v\n\tExpected: %+v", - i + 1, msg, instance.msg) + i+1, msg, instance.msg) } } } - func TestMarshal(t *testing.T) { for i, instance := range codecTest_validInstances { data, err := bencode.Marshal(instance.msg) if err != nil { - t.Errorf("Error while marshalling valid msg #%d: %v", i + 1, err) + t.Errorf("Error while marshalling valid msg #%d: %v", i+1, err) } if bytes.Equal(data, instance.data) != true { t.Errorf("Valid msg #%d marshalled wrong!\n\tGot : %q\n\tExpected: %q", - i + 1, data, instance.data) + i+1, data, instance.data) } } } diff --git a/src/magneticod/dht/mainline/protocol.go b/magneticod/dht/mainline/protocol.go similarity index 89% rename from src/magneticod/dht/mainline/protocol.go rename to magneticod/dht/mainline/protocol.go index c2dd74a..53459fe 100644 --- a/src/magneticod/dht/mainline/protocol.go +++ b/magneticod/dht/mainline/protocol.go @@ -10,16 +10,14 @@ import ( "go.uber.org/zap" ) - type Protocol struct { previousTokenSecret, currentTokenSecret []byte - tokenLock sync.Mutex - transport *Transport - eventHandlers ProtocolEventHandlers - started bool + tokenLock sync.Mutex + transport *Transport + eventHandlers ProtocolEventHandlers + started bool } - type ProtocolEventHandlers struct { OnPingQuery func(*Message, net.Addr) OnFindNodeQuery func(*Message, net.Addr) @@ -30,7 +28,6 @@ type ProtocolEventHandlers struct { OnPingORAnnouncePeerResponse func(*Message, net.Addr) } - func NewProtocol(laddr string, eventHandlers ProtocolEventHandlers) (p *Protocol) { p = new(Protocol) p.transport = NewTransport(laddr, p.onMessage) @@ -46,7 +43,6 @@ func NewProtocol(laddr string, eventHandlers ProtocolEventHandlers) (p *Protocol return } - func (p *Protocol) Start() { if p.started { zap.L().Panic("Attempting to Start() a mainline/Transport that has been already started! (Programmer error.)") @@ -57,12 +53,10 @@ func (p *Protocol) Start() { go p.updateTokenSecret() } - func (p *Protocol) Terminate() { p.transport.Terminate() } - func (p *Protocol) onMessage(msg *Message, addr net.Addr) { switch msg.Y { case "q": @@ -115,23 +109,23 @@ func (p *Protocol) onMessage(msg *Message, addr net.Addr) { } case "r": // get_peers > find_node > ping / announce_peer - if len(msg.R.Token) != 0 { // The message should be a get_peers response. + if len(msg.R.Token) != 0 { // The message should be a get_peers response. if !validateGetPeersResponseMessage(msg) { zap.L().Debug("An invalid get_peers response received!") return } - if p.eventHandlers.OnGetPeersResponse != nil{ + if p.eventHandlers.OnGetPeersResponse != nil { p.eventHandlers.OnGetPeersResponse(msg, addr) } - } else if len(msg.R.Nodes) != 0 { // The message should be a find_node response. + } else if len(msg.R.Nodes) != 0 { // The message should be a find_node response. if !validateFindNodeResponseMessage(msg) { zap.L().Debug("An invalid find_node response received!") return } - if p.eventHandlers.OnFindNodeResponse != nil{ + if p.eventHandlers.OnFindNodeResponse != nil { p.eventHandlers.OnFindNodeResponse(msg, addr) } - } else { // The message should be a ping or an announce_peer response. + } else { // The message should be a ping or an announce_peer response. if !validatePingORannouncePeerResponseMessage(msg) { zap.L().Debug("An invalid ping OR announce_peer response received!") return @@ -141,50 +135,47 @@ func (p *Protocol) onMessage(msg *Message, addr net.Addr) { } } case "e": - zap.L().Sugar().Debugf("Protocol error received: `%s` (%d)", msg.E.Message, msg.E.Code) + // TODO: currently ignoring Server Error 202 + if msg.E.Code != 202 { + zap.L().Sugar().Debugf("Protocol error received: `%s` (%d)", msg.E.Message, msg.E.Code) + } default: /* zap.L().Debug("A KRPC message of an unknown type received!", - zap.String("type", msg.Y)) + zap.String("type", msg.Y)) */ } } - func (p *Protocol) SendMessage(msg *Message, addr net.Addr) { p.transport.WriteMessages(msg, addr) } - func NewPingQuery(id []byte) *Message { panic("Not implemented yet!") } - func NewFindNodeQuery(id []byte, target []byte) *Message { return &Message{ Y: "q", T: []byte("aa"), Q: "find_node", A: QueryArguments{ - ID: id, + ID: id, Target: target, }, } } - func NewGetPeersQuery(id []byte, info_hash []byte) *Message { panic("Not implemented yet!") } - func NewAnnouncePeerQuery(id []byte, implied_port bool, info_hash []byte, port uint16, token []byte) *Message { panic("Not implemented yet!") } - func NewPingResponse(t []byte, id []byte) *Message { return &Message{ Y: "r", @@ -195,36 +186,31 @@ func NewPingResponse(t []byte, id []byte) *Message { } } - func NewFindNodeResponse(t []byte, id []byte, nodes []CompactNodeInfo) *Message { panic("Not implemented yet!") } - func NewGetPeersResponseWithValues(t []byte, id []byte, token []byte, values []CompactPeer) *Message { panic("Not implemented yet!") } - func NewGetPeersResponseWithNodes(t []byte, id []byte, token []byte, nodes []CompactNodeInfo) *Message { return &Message{ Y: "r", T: t, R: ResponseValues{ - ID: id, + ID: id, Token: token, Nodes: nodes, }, } } - func NewAnnouncePeerResponse(t []byte, id []byte) *Message { // Because they are indistinguishable. return NewPingResponse(t, id) } - func (p *Protocol) CalculateToken(address net.IP) []byte { p.tokenLock.Lock() defer p.tokenLock.Unlock() @@ -232,7 +218,6 @@ func (p *Protocol) CalculateToken(address net.IP) []byte { return sum[:] } - func (p *Protocol) VerifyToken(address net.IP, token []byte) bool { p.tokenLock.Lock() defer p.tokenLock.Unlock() @@ -241,7 +226,6 @@ func (p *Protocol) VerifyToken(address net.IP, token []byte) bool { return false } - func (p *Protocol) updateTokenSecret() { for range time.Tick(10 * time.Minute) { p.tokenLock.Lock() @@ -255,24 +239,20 @@ func (p *Protocol) updateTokenSecret() { } } - func validatePingQueryMessage(msg *Message) bool { return len(msg.A.ID) == 20 } - func validateFindNodeQueryMessage(msg *Message) bool { return len(msg.A.ID) == 20 && len(msg.A.Target) == 20 } - func validateGetPeersQueryMessage(msg *Message) bool { return len(msg.A.ID) == 20 && len(msg.A.InfoHash) == 20 } - func validateAnnouncePeerQueryMessage(msg *Message) bool { return len(msg.A.ID) == 20 && len(msg.A.InfoHash) == 20 && @@ -280,7 +260,6 @@ func validateAnnouncePeerQueryMessage(msg *Message) bool { len(msg.A.Token) > 0 } - func validatePingORannouncePeerResponseMessage(msg *Message) bool { return len(msg.R.ID) == 20 } @@ -295,7 +274,6 @@ func validateFindNodeResponseMessage(msg *Message) bool { return true } - func validateGetPeersResponseMessage(msg *Message) bool { return len(msg.R.ID) == 20 && len(msg.R.Token) > 0 diff --git a/src/magneticod/dht/mainline/protocol_test.go b/magneticod/dht/mainline/protocol_test.go similarity index 84% rename from src/magneticod/dht/mainline/protocol_test.go rename to magneticod/dht/mainline/protocol_test.go index abbb0de..586643a 100644 --- a/src/magneticod/dht/mainline/protocol_test.go +++ b/magneticod/dht/mainline/protocol_test.go @@ -1,15 +1,14 @@ package mainline import ( - "testing" "net" + "testing" ) - var protocolTest_validInstances = []struct { validator func(*Message) bool msg Message -} { +}{ // ping Query: { validator: validatePingQueryMessage, @@ -42,7 +41,7 @@ var protocolTest_validInstances = []struct { Y: "q", Q: "find_node", A: QueryArguments{ - ID: []byte("abcdefghij0123456789"), + ID: []byte("abcdefghij0123456789"), Target: []byte("mnopqrstuvwxyz123456"), }, }, @@ -54,7 +53,7 @@ var protocolTest_validInstances = []struct { T: []byte("aa"), Y: "r", R: ResponseValues{ - ID: []byte("0123456789abcdefghij"), + ID: []byte("0123456789abcdefghij"), Nodes: []CompactNodeInfo{}, }, }, @@ -69,7 +68,7 @@ var protocolTest_validInstances = []struct { ID: []byte("0123456789abcdefghij"), Nodes: []CompactNodeInfo{ { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, }, @@ -86,35 +85,35 @@ var protocolTest_validInstances = []struct { ID: []byte("0123456789abcdefghij"), Nodes: []CompactNodeInfo{ { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("zyxwvutsrqponmlkjihg"), + ID: []byte("zyxwvutsrqponmlkjihg"), Addr: net.UDPAddr{IP: []byte("\xf5\x8e\x82\x8b"), Port: 6931, Zone: ""}, }, }, @@ -129,7 +128,7 @@ var protocolTest_validInstances = []struct { Y: "q", Q: "get_peers", A: QueryArguments{ - ID: []byte("abcdefghij0123456789"), + ID: []byte("abcdefghij0123456789"), InfoHash: []byte("mnopqrstuvwxyz123456"), }, }, @@ -141,7 +140,7 @@ var protocolTest_validInstances = []struct { T: []byte("aa"), Y: "r", R: ResponseValues{ - ID: []byte("abcdefghij0123456789"), + ID: []byte("abcdefghij0123456789"), Token: []byte("aoeusnth"), Values: []CompactPeer{ {IP: []byte("axje"), Port: 11893}, @@ -157,15 +156,15 @@ var protocolTest_validInstances = []struct { T: []byte("aa"), Y: "r", R: ResponseValues{ - ID: []byte("abcdefghij0123456789"), + ID: []byte("abcdefghij0123456789"), Token: []byte("aoeusnth"), Nodes: []CompactNodeInfo{ { - ID: []byte("abcdefghijklmnopqrst"), + ID: []byte("abcdefghijklmnopqrst"), Addr: net.UDPAddr{IP: []byte("\x8b\x82\x8e\xf5"), Port: 3169, Zone: ""}, }, { - ID: []byte("zyxwvutsrqponmlkjihg"), + ID: []byte("zyxwvutsrqponmlkjihg"), Addr: net.UDPAddr{IP: []byte("\xf5\x8e\x82\x8b"), Port: 6931, Zone: ""}, }, }, @@ -180,42 +179,38 @@ var protocolTest_validInstances = []struct { Y: "q", Q: "announce_peer", A: QueryArguments{ - ID: []byte("abcdefghij0123456789"), + ID: []byte("abcdefghij0123456789"), InfoHash: []byte("mnopqrstuvwxyz123456"), - Port: 6881, - Token: []byte("aoeusnth"), + Port: 6881, + Token: []byte("aoeusnth"), }, }, }, // TODO: Add announce_peer Query with optional `implied_port` argument. } - func TestValidators(t *testing.T) { for i, instance := range protocolTest_validInstances { if isValid := instance.validator(&instance.msg); !isValid { - t.Errorf("False-positive for valid msg #%d!", i + 1) + t.Errorf("False-positive for valid msg #%d!", i+1) } } } - func TestNewFindNodeQuery(t *testing.T) { if !validateFindNodeQueryMessage(NewFindNodeQuery([]byte("qwertyuopasdfghjklzx"), []byte("xzlkjhgfdsapouytrewq"))) { t.Errorf("NewFindNodeQuery returned an invalid message!") } } - func TestNewPingResponse(t *testing.T) { if !validatePingORannouncePeerResponseMessage(NewPingResponse([]byte("tt"), []byte("qwertyuopasdfghjklzx"))) { t.Errorf("NewPingResponse returned an invalid message!") } } - func TestNewGetPeersResponseWithNodes(t *testing.T) { if !validateGetPeersResponseMessage(NewGetPeersResponseWithNodes([]byte("tt"), []byte("qwertyuopasdfghjklzx"), []byte("token"), []CompactNodeInfo{})) { t.Errorf("NewGetPeersResponseWithNodes returned an invalid message!") } -} \ No newline at end of file +} diff --git a/src/magneticod/dht/mainline/service.go b/magneticod/dht/mainline/service.go similarity index 91% rename from src/magneticod/dht/mainline/service.go rename to magneticod/dht/mainline/service.go index c01e5c1..660a617 100644 --- a/src/magneticod/dht/mainline/service.go +++ b/magneticod/dht/mainline/service.go @@ -1,7 +1,7 @@ package mainline import ( - "crypto/rand" + "math/rand" "net" "sync" "time" @@ -13,14 +13,15 @@ import ( type TrawlingResult struct { InfoHash metainfo.Hash - Peer torrent.Peer + Peer torrent.Peer + PeerIP net.IP + PeerPort int } - type TrawlingService struct { // Private - protocol *Protocol - started bool + protocol *Protocol + started bool eventHandlers TrawlingServiceEventHandlers trueNodeID []byte @@ -28,16 +29,14 @@ type TrawlingService struct { // understandably) slices cannot be used as keys (since they are not hashable), and using arrays // (or even the conversion between each other) is a pain; hence map[string]net.UDPAddr // ^~~~~~ - routingTable map[string]net.Addr + routingTable map[string]net.Addr routingTableMutex *sync.Mutex } - type TrawlingServiceEventHandlers struct { OnResult func(TrawlingResult) } - func NewTrawlingService(laddr string, eventHandlers TrawlingServiceEventHandlers) *TrawlingService { service := new(TrawlingService) service.protocol = NewProtocol( @@ -61,7 +60,6 @@ func NewTrawlingService(laddr string, eventHandlers TrawlingServiceEventHandlers return service } - func (s *TrawlingService) Start() { if s.started { zap.L().Panic("Attempting to Start() a mainline/TrawlingService that has been already started! (Programmer error.)") @@ -74,19 +72,17 @@ func (s *TrawlingService) Start() { zap.L().Info("Trawling Service started!") } - func (s *TrawlingService) Terminate() { s.protocol.Terminate() } - func (s *TrawlingService) trawl() { for range time.Tick(1 * time.Second) { s.routingTableMutex.Lock() if len(s.routingTable) == 0 { s.bootstrap() } else { - zap.L().Info("Latest status:", zap.Int("n", len(s.routingTable))) + zap.L().Debug("Latest status:", zap.Int("n", len(s.routingTable))) s.findNeighbors() s.routingTable = make(map[string]net.Addr) } @@ -94,7 +90,6 @@ func (s *TrawlingService) trawl() { } } - func (s *TrawlingService) bootstrap() { bootstrappingNodes := []string{ "router.bittorrent.com:6881", @@ -119,7 +114,6 @@ func (s *TrawlingService) bootstrap() { } } - func (s *TrawlingService) findNeighbors() { target := make([]byte, 20) for nodeID, addr := range s.routingTable { @@ -135,7 +129,6 @@ func (s *TrawlingService) findNeighbors() { } } - func (s *TrawlingService) onGetPeersQuery(query *Message, addr net.Addr) { s.protocol.SendMessage( NewGetPeersResponseWithNodes( @@ -148,7 +141,6 @@ func (s *TrawlingService) onGetPeersQuery(query *Message, addr net.Addr) { ) } - func (s *TrawlingService) onAnnouncePeerQuery(query *Message, addr net.Addr) { var peerPort int if query.A.ImpliedPort != 0 { @@ -166,8 +158,8 @@ func (s *TrawlingService) onAnnouncePeerQuery(query *Message, addr net.Addr) { InfoHash: infoHash, Peer: torrent.Peer{ // As we don't know the ID of the remote peer, set it empty. - Id: peerId, - IP: addr.(*net.UDPAddr).IP, + Id: peerId, + IP: addr.(*net.UDPAddr).IP, Port: peerPort, // "Ha" indicates that we discovered the peer through DHT Announce Peer (query); not // sure how anacrolix/torrent utilizes that information though. @@ -176,6 +168,8 @@ func (s *TrawlingService) onAnnouncePeerQuery(query *Message, addr net.Addr) { // that it doesn't. SupportsEncryption: false, }, + PeerIP: addr.(*net.UDPAddr).IP, + PeerPort: peerPort, }) s.protocol.SendMessage( @@ -187,14 +181,13 @@ func (s *TrawlingService) onAnnouncePeerQuery(query *Message, addr net.Addr) { ) } - func (s *TrawlingService) onFindNodeResponse(response *Message, addr net.Addr) { s.routingTableMutex.Lock() defer s.routingTableMutex.Unlock() for _, node := range response.R.Nodes { - if node.Addr.Port != 0 { // Ignore nodes who "use" port 0. - if len(s.routingTable) < 10000 { + if node.Addr.Port != 0 { // Ignore nodes who "use" port 0. + if len(s.routingTable) < 8000 { s.routingTable[string(node.ID)] = &node.Addr } } diff --git a/src/magneticod/dht/mainline/transport.go b/magneticod/dht/mainline/transport.go similarity index 96% rename from src/magneticod/dht/mainline/transport.go rename to magneticod/dht/mainline/transport.go index eaa2f4b..8bc7321 100644 --- a/src/magneticod/dht/mainline/transport.go +++ b/magneticod/dht/mainline/transport.go @@ -3,12 +3,11 @@ package mainline import ( "net" - "go.uber.org/zap" "github.com/anacrolix/torrent/bencode" + "go.uber.org/zap" "strings" ) - type Transport struct { conn *net.UDPConn laddr *net.UDPAddr @@ -20,11 +19,11 @@ type Transport struct { onMessage func(*Message, net.Addr) } - -func NewTransport(laddr string, onMessage func(*Message, net.Addr)) (*Transport) { +func NewTransport(laddr string, onMessage func(*Message, net.Addr)) *Transport { transport := new(Transport) transport.onMessage = onMessage - var err error; transport.laddr, err = net.ResolveUDPAddr("udp", laddr) + var err error + transport.laddr, err = net.ResolveUDPAddr("udp", laddr) if err != nil { zap.L().Panic("Could not resolve the UDP address for the trawler!", zap.Error(err)) } @@ -32,7 +31,6 @@ func NewTransport(laddr string, onMessage func(*Message, net.Addr)) (*Transport) return transport } - func (t *Transport) Start() { // Why check whether the Transport `t` started or not, here and not -for instance- in // t.Terminate()? @@ -56,12 +54,10 @@ func (t *Transport) Start() { go t.readMessages() } - func (t *Transport) Terminate() { t.conn.Close() } - // readMessages is a goroutine! func (t *Transport) readMessages() { buffer := make([]byte, 65536) @@ -87,7 +83,6 @@ func (t *Transport) readMessages() { } } - func (t *Transport) WriteMessages(msg *Message, addr net.Addr) { data, err := bencode.Marshal(msg) if err != nil { diff --git a/src/magneticod/dht/mainline/transport_test.go b/magneticod/dht/mainline/transport_test.go similarity index 99% rename from src/magneticod/dht/mainline/transport_test.go rename to magneticod/dht/mainline/transport_test.go index e811a7f..bf441ee 100644 --- a/src/magneticod/dht/mainline/transport_test.go +++ b/magneticod/dht/mainline/transport_test.go @@ -6,7 +6,6 @@ import ( "testing" ) - func TestReadFromOnClosedConn(t *testing.T) { // Initialization laddr, err := net.ResolveUDPAddr("udp", "0.0.0.0:0") @@ -31,7 +30,6 @@ func TestReadFromOnClosedConn(t *testing.T) { } } - func TestWriteToOnClosedConn(t *testing.T) { // Initialization laddr, err := net.ResolveUDPAddr("udp", "0.0.0.0:0") @@ -52,4 +50,4 @@ func TestWriteToOnClosedConn(t *testing.T) { if !(err != nil && strings.HasSuffix(err.Error(), "use of closed network connection")) { t.Fatalf("Unexpected suffix in the error message!") } -} \ No newline at end of file +} diff --git a/src/magneticod/dht/managers.go b/magneticod/dht/managers.go similarity index 91% rename from src/magneticod/dht/managers.go rename to magneticod/dht/managers.go index 1dbbba6..3c213a9 100644 --- a/src/magneticod/dht/managers.go +++ b/magneticod/dht/managers.go @@ -1,15 +1,13 @@ package dht -import "magneticod/dht/mainline" - +import "magnetico/magneticod/dht/mainline" type TrawlingManager struct { // private - output chan mainline.TrawlingResult + output chan mainline.TrawlingResult services []*mainline.TrawlingService } - func NewTrawlingManager(mlAddrs []string) *TrawlingManager { manager := new(TrawlingManager) manager.output = make(chan mainline.TrawlingResult) @@ -33,17 +31,14 @@ func NewTrawlingManager(mlAddrs []string) *TrawlingManager { return manager } - func (m *TrawlingManager) onResult(res mainline.TrawlingResult) { m.output <- res } - func (m *TrawlingManager) Output() <-chan mainline.TrawlingResult { return m.output } - func (m *TrawlingManager) Terminate() { for _, service := range m.services { service.Terminate() diff --git a/magneticod/main.go b/magneticod/main.go new file mode 100644 index 0000000..9a27a11 --- /dev/null +++ b/magneticod/main.go @@ -0,0 +1,157 @@ +package main + +import ( + "fmt" + "net" + "os" + "os/signal" + "time" + + "github.com/jessevdk/go-flags" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "magnetico/magneticod/bittorrent" + "magnetico/magneticod/dht" + + "magnetico/persistence" +) + +type cmdFlags struct { + DatabaseURL string `long:"database" description:"URL of the database." required:"yeah"` + + TrawlerMlAddrs []string `long:"trawler-ml-addr" description:"Address(es) to be used by trawling DHT (Mainline) nodes." default:"0.0.0.0:0"` + TrawlerMlInterval uint `long:"trawler-ml-interval" description:"Trawling interval in integer deciseconds (one tenth of a second)."` + + Verbose []bool `short:"v" long:"verbose" description:"Increases verbosity."` +} + +type opFlags struct { + DatabaseURL string + + TrawlerMlAddrs []string + TrawlerMlInterval time.Duration + + Verbosity int +} + +func main() { + loggerLevel := zap.NewAtomicLevel() + // Logging levels: ("debug", "info", "warn", "error", "dpanic", "panic", and "fatal"). + logger := zap.New(zapcore.NewCore( + zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), + zapcore.Lock(os.Stderr), + loggerLevel, + )) + defer logger.Sync() + zap.ReplaceGlobals(logger) + + // opFlags is the "operational flags" + opFlags, err := parseFlags() + if err != nil { + // Do not print any error messages as jessevdk/go-flags already did. + return + } + + zap.L().Info("magneticod v0.7.0 has been started.") + zap.L().Info("Copyright (C) 2017 Mert Bora ALPER .") + zap.L().Info("Dedicated to Cemile Binay, in whose hands I thrived.") + + switch opFlags.Verbosity { + case 0: + loggerLevel.SetLevel(zap.WarnLevel) + case 1: + loggerLevel.SetLevel(zap.InfoLevel) + default: // Default: i.e. in case of 2 or more. + // TODO: print the caller (function)'s name and line number! + loggerLevel.SetLevel(zap.DebugLevel) + } + + zap.ReplaceGlobals(logger) + + // Handle Ctrl-C gracefully. + interruptChan := make(chan os.Signal) + signal.Notify(interruptChan, os.Interrupt) + + database, err := persistence.MakeDatabase(opFlags.DatabaseURL, false, logger) + if err != nil { + logger.Sugar().Fatalf("Could not open the database at `%s`: %s", opFlags.DatabaseURL, err.Error()) + } + + trawlingManager := dht.NewTrawlingManager(opFlags.TrawlerMlAddrs) + metadataSink := bittorrent.NewMetadataSink(2 * time.Minute) + + // The Event Loop + for stopped := false; !stopped; { + select { + case result := <-trawlingManager.Output(): + logger.Info("result: ", zap.String("hash", result.InfoHash.String())) + exists, err := database.DoesTorrentExist(result.InfoHash[:]) + if err != nil { + zap.L().Fatal("Could not check whether torrent exists!", zap.Error(err)) + } else if !exists { + metadataSink.Sink(result) + } + + case metadata := <-metadataSink.Drain(): + if err := database.AddNewTorrent(metadata.InfoHash, metadata.Name, metadata.Files); err != nil { + logger.Sugar().Fatalf("Could not add new torrent %x to the database: %s", + metadata.InfoHash, err.Error()) + } + logger.Sugar().Infof("D I S C O V E R E D: `%s` %x", metadata.Name, metadata.InfoHash) + + case <-interruptChan: + trawlingManager.Terminate() + stopped = true + } + } + + if err = database.Close(); err != nil { + zap.L().Error("Could not close database!", zap.Error(err)) + } +} + +func parseFlags() (*opFlags, error) { + opF := new(opFlags) + cmdF := new(cmdFlags) + + _, err := flags.Parse(cmdF) + if err != nil { + return nil, err + } + + if cmdF.DatabaseURL == "" { + zap.S().Fatal("database") + } else { + opF.DatabaseURL = cmdF.DatabaseURL + } + + if err = checkAddrs(cmdF.TrawlerMlAddrs); err != nil { + zap.S().Fatalf("Of argument (list) `trawler-ml-addr` %s", err.Error()) + } else { + opF.TrawlerMlAddrs = cmdF.TrawlerMlAddrs + } + + // 1 decisecond = 100 milliseconds = 0.1 seconds + if cmdF.TrawlerMlInterval == 0 { + opF.TrawlerMlInterval = time.Duration(1) * 100 * time.Millisecond + } else { + opF.TrawlerMlInterval = time.Duration(cmdF.TrawlerMlInterval) * 100 * time.Millisecond + } + + opF.Verbosity = len(cmdF.Verbose) + + return opF, nil +} + +func checkAddrs(addrs []string) error { + for i, addr := range addrs { + // We are using ResolveUDPAddr but it works equally well for checking TCPAddr(esses) as + // well. + _, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return fmt.Errorf("with %d(th) address `%s`: %s", i+1, addr, err.Error()) + } + } + return nil +} diff --git a/src/magneticod/main_test.go b/magneticod/main_test.go similarity index 99% rename from src/magneticod/main_test.go rename to magneticod/main_test.go index d67e761..4c2c76d 100644 --- a/src/magneticod/main_test.go +++ b/magneticod/main_test.go @@ -21,4 +21,3 @@ func TestAppdirs(t *testing.T) { t.Errorf("UserCacheDir returned an unexpected value! `%s`", returned) } } - diff --git a/magneticow/.dockerignore b/magneticow/.dockerignore deleted file mode 100644 index 4a246ec..0000000 --- a/magneticow/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -Dockerfile -.dockerignore diff --git a/magneticow/Dockerfile b/magneticow/Dockerfile deleted file mode 100644 index 2942164..0000000 --- a/magneticow/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.6 - -RUN mkdir -p /usr/src/app - -WORKDIR /usr/src/app - -COPY . . -RUN pip install -e . -EXPOSE 8080 -CMD ["python", "-mmagneticow", "--port", "8080", "--user", "user", "password"] diff --git a/magneticow/MANIFEST.in b/magneticow/MANIFEST.in deleted file mode 100644 index 61d4d26..0000000 --- a/magneticow/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -recursive-include magneticow/static * -recursive-include magneticow/templates * diff --git a/magneticow/README.rst b/magneticow/README.rst deleted file mode 100644 index 3056998..0000000 --- a/magneticow/README.rst +++ /dev/null @@ -1,205 +0,0 @@ -========== -magneticow -========== -*Lightweight web interface for magnetico.* - -**magneticow** is a lightweight web interface to search and to browse the torrents that its counterpart (**magneticod**) -discovered. It allows fast full text search of the names of the torrents, by correctly parsing them into their elements. - -Installation -============ -**magneticow** uses `gevent `_ as a "standalone WSGI container" (you can think of it as an -embedded HTTP server), and connects to the same SQLite 3 database that **magneticod** writes. Hence, **root or sudo -access is NOT required at any stage, during or after the installation process.** - -Requirements ------------- -- Python 3.5 or above. - -Instructions ------------- - **WARNING:** - - **magnetico** currently does NOT have any filtering system NOR it allows individual torrents to be removed from the - database, and BitTorrent DHT network is full of the materials that are considered illegal in many countries - (violence, pornography, copyright infringing content, and even child-pornography). If you are afraid of the legal - consequences, or simply morally against (indirectly) assisting those content to spread around, follow the - **magneticow** installation instructions carefully to password-protect the web-interface from others. -\ - - **WARNING:** - - **magneticow** is *NOT* designed to scale, and will fail miserably if you try to use it like a public torrent - website. This is a *deliberate* technical decision, not a bug or something to be fixed; another web interface with - more features to support such use cases and scalability *might* be developed, but **magneticow** will NEVER be the - case. - -1. Download the latest version of **magneticow** from PyPI: :: - - pip3 install magneticow --user - -2. Add installation path to the ``$PATH``; append the following line to your ``~/.profile`` if you are using bash - *(you can skip to step 4 if you installed magneticod first as advised)* :: - - export PATH=$PATH:~/.local/bin - - **or if you are on macOS**, (assuming that you are using Python 3.5): :: - - export PATH="${PATH}:${HOME}/Library/Python/3.5/bin/" - -3. Activate the changes to ``$PATH`` (again, if you are using bash): :: - - source ~/.profile - -4. Confirm that it is running: :: - - magneticow --port 8080 --user username_1 password_1 --user username_2 password_2 - - Do not forget to actually visit the website, and run a search without any keywords (i.e. simply press the enter - button); this should return a list of most recently discovered torrents. If **magneticod** has not been running long - enough, database might be completely empty and you might see no results (5 minutes should suffice to discover more - than a dozen torrents). - -5. *(only for systemd users, skip the rest of the steps and proceed to the* `Using`_ *section if you are not a systemd - user or want to use a different solution)* - - Download the magneticow systemd service file (at - `magneticow/systemd/magneticow.service `_) and expand the tilde symbol with the path of - your home directory. Also, choose a port (> 1024) for **magneticow** to listen on, and supply username(s) and - password(s). - - For example, if my home directory is ``/home/bora``, and I want to create two users named ``bora`` and ``tolga`` with - passwords ``staatsangehörigkeit`` and ``bürgerschaft``, and then **magneticow** to listen on port 8080, this line :: - - ExecStart=~/.local/bin/magneticow --port PORT --user USERNAME PASSWORD - - should become this: :: - - ExecStart=/home/bora/.local/bin/magneticow --port 8080 --user bora staatsangehörigkeit --user tolga bürgerschaft - - Run ``echo ~`` to see the path of your own home directory, if you do not already know. - - **WARNING:** - - **At least one username and password MUST be supplied.** This is to protect the privacy of your **magneticow** - installation, although **beware that this does NOT encrypt the communication between your browser and the - server!** - -6. Copy the magneticow systemd service file to your local systemd configuration directory: :: - - cp magneticow.service ~/.config/systemd/user/ - -7. Start **magneticow**: :: - - systemctl --user enable magneticow --now - - **magneticow** should now be running under the supervision of systemd and it should also be automatically started - whenever you boot your machine. - - You can check its status and most recent log entries using the following command: :: - - systemctl --user status magneticow - - To stop **magneticow**, issue the following: :: - - systemctl --user stop magneticow - -Using -===== -**magneticow** does not require user interference to operate, once it starts running. Hence, there is no "user manual", -although you should beware of these points: - -1. **Resource Usage:** - - To repeat it for the last time, **magneticow** is a lightweight web interface for magnetico and is not suitable for - handling many users simultaneously. Misusing **magneticow** will likely to lead high processor usage and increased - response times. If that is the case, you might consider lowering the priority of **magneticow** using ``renice`` - command. - -2. **Pre-Alpha Bugs:** - - **magneticow** is *supposed* to work "just fine", but as being at pre-alpha stage, it's likely that you might find - some bugs. It will be much appreciated if you can report those bugs, so that **magneticow** can be improved. See the - next sub-section for how to mitigate the issue if you are *not* using systemd. - -Automatic Restarting --------------------- -Due to minor bugs at this stage of its development, **magneticow** should be supervised by another program to be ensured -that it's running, and should be restarted if not. systemd service file supplied by **magneticow** implements that, -although (if you wish) you can also use a much more primitive approach using GNU screen (which comes pre-installed in -many GNU/Linux distributions): - -1. Start screen session named ``magneticow``: :: - - screen -S magneticow - -2. Run **magneticow** forever: :: - - until magneticow; do echo "restarting..."; sleep 5; done; - - This will keep restarting **magneticow** after five seconds in case if it fails. - -3. Detach the session by pressing Ctrl+A and after Ctrl+D. - -4. If you wish to see the logs, or to kill **magneticow**, ``screen -r magneticow`` will attach the original screen - session back. **magneticow** will exit gracefully upon keyboard interrupt (Ctrl+C) [SIGINT]. - -Searching ---------- -* Only the **titles** of the torrents are being searched. -* Search is case-insensitive. -* Titles that includes terms that are separated by space are returned from the search: - - Example: ``king bad`` returns ``Stephen King - The Bazaar of Bad Dreams`` - - * If you would like terms to appear in the exact order you wrote them, enclose them in double quotes: - - Example: ``"king bad"`` returns ``George Benson - Good King Bad`` -* Use asteriks (``*``) to denote prefixes: - - Example: ``The Godf*`` returns ``Francis Ford Coppola - The Godfather`` - - Asteriks works inside the double quotes too! -* Use caret (``^``) to indicate that the term it prefixes must be the first word in the title: - - Example: ``linux`` returns ``Arch Linux`` while ``^linux`` would return ``Linux Mint`` - - * Caret works **inside** the double quotes **but not outside**: - - Right: ``"^ubuntu linux"`` - - Wrong: ``^"ubuntu linux"`` -* You can use ``AND``, ``OR`` and ``NOT`` and also parentheses for more complex queries: - - Example: ``programming NOT (windows OR "os x" OR macos)`` - - Beware that the terms are all-caps and MUST be so. - - ======================= ======================================= - Operator Enhanced Query Syntax Precedence - ======================= ======================================= - NOT Highest precedence (tightest grouping). - AND - OR Lowest precedence (loosest grouping). - ======================= ======================================= - -REST-ful HTTP API -================= - **magneticow** offers a REST-ful HTTP API for 3rd-party applications to interact with **magnetico** setups. Examples - would be an Android app for searching torrents **magnetico** discovered and queueing them on your seedbox, or a - custom data analysis/statistics application developed for a research project on BitTorrent network. Nevertheless, it - is up to you what to do with it at the end of the day. - -See `API documentation <./docs/API/README.md>`_ for more details. - -License -======= -All the code is licensed under AGPLv3, unless otherwise stated in the source specific source. See ``COPYING`` file -in ``magnetico`` directory for the full license text. - - ----- - -Dedicated to Cemile Binay, in whose hands I thrived. - -Bora M. ALPER diff --git a/src/magneticow/data/static/assets/feed.png b/magneticow/data/static/assets/feed.png similarity index 100% rename from src/magneticow/data/static/assets/feed.png rename to magneticow/data/static/assets/feed.png diff --git a/src/magneticow/data/static/assets/magnet.gif b/magneticow/data/static/assets/magnet.gif similarity index 100% rename from src/magneticow/data/static/assets/magnet.gif rename to magneticow/data/static/assets/magnet.gif diff --git a/src/magneticow/data/static/fonts/NotoMono/LICENSE_OFL.txt b/magneticow/data/static/fonts/NotoMono/LICENSE_OFL.txt similarity index 100% rename from src/magneticow/data/static/fonts/NotoMono/LICENSE_OFL.txt rename to magneticow/data/static/fonts/NotoMono/LICENSE_OFL.txt diff --git a/src/magneticow/data/static/fonts/NotoMono/Regular.ttf b/magneticow/data/static/fonts/NotoMono/Regular.ttf similarity index 100% rename from src/magneticow/data/static/fonts/NotoMono/Regular.ttf rename to magneticow/data/static/fonts/NotoMono/Regular.ttf diff --git a/src/magneticow/data/static/fonts/NotoSansUI/Bold.ttf b/magneticow/data/static/fonts/NotoSansUI/Bold.ttf similarity index 100% rename from src/magneticow/data/static/fonts/NotoSansUI/Bold.ttf rename to magneticow/data/static/fonts/NotoSansUI/Bold.ttf diff --git a/src/magneticow/data/static/fonts/NotoSansUI/BoldItalic.ttf b/magneticow/data/static/fonts/NotoSansUI/BoldItalic.ttf similarity index 100% rename from src/magneticow/data/static/fonts/NotoSansUI/BoldItalic.ttf rename to magneticow/data/static/fonts/NotoSansUI/BoldItalic.ttf diff --git a/src/magneticow/data/static/fonts/NotoSansUI/Italic.ttf b/magneticow/data/static/fonts/NotoSansUI/Italic.ttf similarity index 100% rename from src/magneticow/data/static/fonts/NotoSansUI/Italic.ttf rename to magneticow/data/static/fonts/NotoSansUI/Italic.ttf diff --git a/src/magneticow/data/static/fonts/NotoSansUI/LICENSE_OFL.txt b/magneticow/data/static/fonts/NotoSansUI/LICENSE_OFL.txt similarity index 100% rename from src/magneticow/data/static/fonts/NotoSansUI/LICENSE_OFL.txt rename to magneticow/data/static/fonts/NotoSansUI/LICENSE_OFL.txt diff --git a/src/magneticow/data/static/fonts/NotoSansUI/Regular.ttf b/magneticow/data/static/fonts/NotoSansUI/Regular.ttf similarity index 100% rename from src/magneticow/data/static/fonts/NotoSansUI/Regular.ttf rename to magneticow/data/static/fonts/NotoSansUI/Regular.ttf diff --git a/src/magneticow/data/static/scripts/plotly-v1.26.1.min.js b/magneticow/data/static/scripts/plotly-v1.26.1.min.js similarity index 100% rename from src/magneticow/data/static/scripts/plotly-v1.26.1.min.js rename to magneticow/data/static/scripts/plotly-v1.26.1.min.js diff --git a/src/magneticow/data/static/scripts/statistics.js b/magneticow/data/static/scripts/statistics.js similarity index 100% rename from src/magneticow/data/static/scripts/statistics.js rename to magneticow/data/static/scripts/statistics.js diff --git a/src/magneticow/data/static/scripts/torrent.js b/magneticow/data/static/scripts/torrent.js similarity index 100% rename from src/magneticow/data/static/scripts/torrent.js rename to magneticow/data/static/scripts/torrent.js diff --git a/src/magneticow/data/static/styles/essential.css b/magneticow/data/static/styles/essential.css similarity index 94% rename from src/magneticow/data/static/styles/essential.css rename to magneticow/data/static/styles/essential.css index 6a92c4e..6c940cb 100644 --- a/src/magneticow/data/static/styles/essential.css +++ b/magneticow/data/static/styles/essential.css @@ -39,7 +39,7 @@ html { } pre { - font-family: 'Noto Mono', monospace; + font-family: 'Noto Mono'; line-height: 1.2em; } @@ -49,12 +49,6 @@ body { line-height: 1.45; } -@media (max-width: 616px) { - body { - padding: 1em 8px 1em 8px; - } -} - b { font-weight: bold; } diff --git a/magneticow/data/static/styles/homepage.css b/magneticow/data/static/styles/homepage.css new file mode 100644 index 0000000..384d922 --- /dev/null +++ b/magneticow/data/static/styles/homepage.css @@ -0,0 +1,27 @@ +main { + display: flex; + align-items: center; + align-content: center; + + height: calc(100vh - 2 * 3em); + width: calc(100vw - 2 * 3em); +} + +main form { + max-width: 600px; + width: 100%; + + margin-left: 0.5em; +} + +main form input { + width: 100%; +} + +main > div { + margin-right: 0.5em; +} + +footer { + margin-top: 0.833em; +} diff --git a/src/magneticow/data/static/styles/reset.css b/magneticow/data/static/styles/reset.css similarity index 100% rename from src/magneticow/data/static/styles/reset.css rename to magneticow/data/static/styles/reset.css diff --git a/src/magneticow/data/static/styles/statistics.css b/magneticow/data/static/styles/statistics.css similarity index 100% rename from src/magneticow/data/static/styles/statistics.css rename to magneticow/data/static/styles/statistics.css diff --git a/src/magneticow/data/static/styles/torrent.css b/magneticow/data/static/styles/torrent.css similarity index 100% rename from src/magneticow/data/static/styles/torrent.css rename to magneticow/data/static/styles/torrent.css diff --git a/src/magneticow/data/static/styles/torrents.css b/magneticow/data/static/styles/torrents.css similarity index 100% rename from src/magneticow/data/static/styles/torrents.css rename to magneticow/data/static/styles/torrents.css diff --git a/magneticow/data/templates/feed.xml b/magneticow/data/templates/feed.xml new file mode 100644 index 0000000..8dafe39 --- /dev/null +++ b/magneticow/data/templates/feed.xml @@ -0,0 +1,13 @@ + + + + {{ .Title }} + {{ range .Items }} + + {{ .Title }} + {{ .InfoHash }} + + + {{ end }} + + \ No newline at end of file diff --git a/src/magneticow/data/templates/homepage.html b/magneticow/data/templates/homepage.html similarity index 70% rename from src/magneticow/data/templates/homepage.html rename to magneticow/data/templates/homepage.html index cbe3872..123bc95 100644 --- a/src/magneticow/data/templates/homepage.html +++ b/magneticow/data/templates/homepage.html @@ -2,7 +2,6 @@ - magneticow @@ -11,14 +10,14 @@
-
magneticow​(pre-alpha)
+
magneticow​(pre-alpha)
- ~{{ . }} torrents available (see the statistics). + ~{{ "{:,}".format(n_torrents) }} torrents available (see the statistics).
diff --git a/src/magneticow/data/templates/statistics.html b/magneticow/data/templates/statistics.html similarity index 50% rename from src/magneticow/data/templates/statistics.html rename to magneticow/data/templates/statistics.html index 7afde2d..9df4ab9 100644 --- a/src/magneticow/data/templates/statistics.html +++ b/magneticow/data/templates/statistics.html @@ -3,11 +3,11 @@ Statistics - magneticow - - - - - + + + + +
diff --git a/src/magneticow/data/templates/torrent.html b/magneticow/data/templates/torrent.html similarity index 75% rename from src/magneticow/data/templates/torrent.html rename to magneticow/data/templates/torrent.html index d44271b..fec83cf 100644 --- a/src/magneticow/data/templates/torrent.html +++ b/magneticow/data/templates/torrent.html @@ -3,10 +3,10 @@ {{ torrent.name }} - magnetico - - - - + + + +
@@ -19,7 +19,7 @@

{{ torrent.name }}

- Magnet link {{ torrent.info_hash }} diff --git a/magneticow/data/templates/torrents.html b/magneticow/data/templates/torrents.html new file mode 100644 index 0000000..2f2386c --- /dev/null +++ b/magneticow/data/templates/torrents.html @@ -0,0 +1,90 @@ + + + + + {% if search %}"{{search}}"{% else %}Most recent torrents{% endif %} - magneticow + + + + + + +
+
magneticow​(pre-alpha)
+
+ +
+ +
+
+ + + + + + + + + + + {% for torrent in torrents %} + + + + + + + {% endfor %} + +
+ {% if sorted_by == "name ASC" %} + Name â–² + {% elif sorted_by == "name DESC" %} + Name â–¼ + {% else %} + Name + {% endif %} + + {% if sorted_by == "total_size ASC" %} + Size â–² + {% elif sorted_by == "total_size DESC" %} + Size â–¼ + {% else %} + Size + {% endif %} + + {% if sorted_by == "discovered_on ASC" %} + Discovered on â–² + {% elif sorted_by == "discovered_on DESC" %} + Discovered on â–¼ + {% else %} + Discovered on + {% endif %} +
+ Magnet link{{ torrent.name }}{{ torrent.size }}{{ torrent.discovered_on }}
+
+
+ +
+ + + {% if sorted_by %} + + {% endif %} + +
+
+ + + {% if sorted_by %} + + {% endif %} + +
+
+ + \ No newline at end of file diff --git a/magneticow/docs/API/README.md b/magneticow/docs/API/README.md deleted file mode 100644 index 3a30a66..0000000 --- a/magneticow/docs/API/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# magneticow API Documentation - -**magneticow** offers a REST-ful HTTP API for 3rd-party applications to interact with **magnetico** setups. Examples -would be an Android app for searching torrents **magnetico** discovered and queueing them on your seedbox, or a custom -data analysis/statistics application developed for a research project on BitTorrent network. Nevertheless, it is up to -you what to do with it at the end of the day. - -The rules stated above below to the API as a whole and across the all versions: - -* The API root is `/api`. -* Right after the API root MUST come the API version in the format `vX` (*e.g.* `/api/v1`). -* Different API versions MAY be backwards-incompatible, but any changes within the same version of the API MUST NOT - break the backwards-compatibility. -* Version 0 (zero) of the API is considered to be experimental and MAY be backwards-incompatible. -* API documentation MUST be considered as a contract between the developers of **magnetico** and **magneticow**, and of - 3rd party application developers, and MUST be respected as such. - -The documentation for the API is organised as described below: - -* Each version of the API MUST be documented in a separate document named `vX.md`. Everything (*i.e.* each - functionality, status codes, etc.) MUST be clearly indicated when they are introduced. -* Each document MUST clearly indicate at the beginning whether it is *finalized* or not. Not-finalised documents (called - *Draft*) CAN be changed, and finalised , but once finalised documents MUST NOT be modified afterwards. -* Documentation for the version 0 (zero) of the API MUST be considered free from the rules above, and always considered - a *draft*. -* Each document MUST be self-standing, that is, MUST be completely understandable and unambiguous without requiring to - refer another document. - * Hence, use quotations when necessary and reference them. - -Remarks: - -* Use British English, and serial comma. -* Documents should be formatted in GitHub Flavoured Markdown. \ No newline at end of file diff --git a/magneticow/docs/API/v0.md b/magneticow/docs/API/v0.md deleted file mode 100644 index 7abc1fc..0000000 --- a/magneticow/docs/API/v0.md +++ /dev/null @@ -1,93 +0,0 @@ -# magneticow API v0 Documentation - -__Status:__ Draft (NOT Finalised) - -__Last Updated:__ 13 June 2017 _by_ Bora M. Alper. - -## General remarks -* All requests MUST be encoded in UTF-8 and same applies for responses too. - * Clients MUST set `Content-type` header to ` application/json; charset=utf-8` for all their requests. -* All dates MUST be in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format, same applies for responses too. - - - -## Collections -* `/torrents`, representing all the torrents in the database. -* `/statistics`, representing all the statistical data about the database. - - -## Methods defined on `/torrents` collection -### GET -__Parameters:__ -* **`query`:** string to be queried in the torrent titles. If absent, all torrents will be returned. - * __ATTENTION:__ `query` MUST NOT be an empty string, if supplied. - * __Remark:__ The format of the `query` is to be clarified! [TODO] -* **`limit`:** number of results to be returned per page. If absent, it's by default 20. -* **`sort_by`:** string enum, MUST be one of the strings `discovered_on`, `relevance` (if `query` is non-empty), `size`, - separated by space, and followed by one of the strings `ASC` or `DESC`, for ascending and descending, respectively. - * __ATTENTION:__ If `sort_by` is `relevance`, `query` is MUST be supplied. - -__Response:__ -* __Status code:__ `200 OK` - -```json -[ - { - "info_hash": "ABCDEFABCDEFABCDEFAB", - "title": "EXAMPLE TITLE", - "discovered_on": "2017-06-13T14:06:01Z", - "files": [ - {"path": "file.ext", "size": 1024}, - {"path": "directory/file_2.ext", "size": 2048}, - {"path": "directory/file_3.ext", "size": 4096}, - ... - ] - }, - ... -] -``` - -* `info_hash` is a hex-encoded info hash of the torrent. -* `discovered_on` is the date the torrent is discovered on. - * __ATTENTION:__ Due to ambiguities about `time()` function in the C standard library, the effect of leap seconds, and - (being pedantic) even the epoch is **platform-dependent**. (The epoch is quite often 1970-01-01T00:00:00Z.) -* `files` is a list of files denoted by their relative-paths. `/` character (U+002F) is used as a path separator, hence - it can be safely assumed that none of the directory or file name can contain it. `\0` (U+0000) is also prohibited to - appear _anywhere_ in the path. - * __Remark:__ These restrictions are not in the BitTorrent specifications. So how **magnetico** enforces that? Well, - **magneticod** simmply ignores torrents with *illegal* file names! - -## Methods defined on `/statistics` collection - -### GET -__Parameters:__ -* **`group_by`:** is how data-points are grouped by; MUST be one of the strings `hour`, `day`, `week`, `month`, or - `year`. -* **`period`:** is two dates, separated by a space chaacter (U+0020), denoting start and end, both inclusive. - * __ATTENTION:__ Depending on the `group_by`, `datetime` WILL be in one of the following formats: - - `yyyy` for `year` (_e.g._ `2017`) - - `yyyy-mm` for `month` (_e.g._ `2017-06`) - - `yyyy-Ww` for `week` (_e.g._ `2017-W25`) - - `yyyy-mm-dd` for `day` (_e.g._ `2017-06-04`) - - `yyyy-mm-ddThh` for `hour` (_e.g._ `2017-06-04:02`) - - -__Response:__ -* __Status code:__ `200 OK` - -```json -[ - { - "datetime": "2017-06", - "new_torrents": 2591558 - }, - { - "datetime": "2017-07", - "new_torrents": 3448754 - }, - ... -] -``` - -* `datetime` is the date (and if applicable, time) of a data-point. - * __Remark:__ Depending on the `group_by`, `datetime` WILL have the *same* format with `period`. diff --git a/magneticow/magneticow/__init__.py b/magneticow/magneticow/__init__.py deleted file mode 100644 index 1234a18..0000000 --- a/magneticow/magneticow/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# magneticow - Lightweight web interface for magnetico. -# Copyright (C) 2017 Mert Bora ALPER -# Dedicated to Cemile Binay, in whose hands I thrived. -# -# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General -# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more -# details. -# -# You should have received a copy of the GNU Affero General Public License along with this program. If not, see -# . diff --git a/magneticow/magneticow/__main__.py b/magneticow/magneticow/__main__.py deleted file mode 100644 index 369032c..0000000 --- a/magneticow/magneticow/__main__.py +++ /dev/null @@ -1,93 +0,0 @@ -# magneticow - Lightweight web interface for magnetico. -# Copyright (C) 2017 Mert Bora ALPER -# Dedicated to Cemile Binay, in whose hands I thrived. -# -# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General -# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more -# details. -# -# You should have received a copy of the GNU Affero General Public License along with this program. If not, see -# . - -import argparse -import logging -import sys -import textwrap - -import gevent.wsgi - -from magneticow import magneticow - - -def main() -> int: - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s") - - arguments = parse_args() - magneticow.app.arguments = arguments - - http_server = gevent.wsgi.WSGIServer((arguments.host, arguments.port), magneticow.app) - - magneticow.initialize_magneticod_db() - - try: - logging.info("magneticow is ready to serve!") - http_server.serve_forever() - except KeyboardInterrupt: - return 0 - finally: - magneticow.close_db() - - return 1 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Lightweight web interface for magnetico.", - epilog=textwrap.dedent("""\ - Copyright (C) 2017 Mert Bora ALPER - Dedicated to Cemile Binay, in whose hands I thrived. - - This program is free software: you can redistribute it and/or modify it under - the terms of the GNU Affero General Public License as published by the Free - Software Foundation, either version 3 of the License, or (at your option) any - later version. - - This program is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - PARTICULAR PURPOSE. See the GNU Affero General Public License for more - details. - - You should have received a copy of the GNU Affero General Public License along - with this program. If not, see . - """), - allow_abbrev=False, - formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument( - "--host", action="store", type=str, required=False, default="", - help="the host address magneticow web server should listen on" - ) - parser.add_argument( - "--port", action="store", type=int, required=True, - help="the port number magneticow web server should listen on" - ) - - auth_group = parser.add_mutually_exclusive_group(required=True) - - auth_group.add_argument( - "--no-auth", dest='noauth', action="store_true", default=False, - help="make the web interface available without authentication" - ) - auth_group.add_argument( - "--user", action="append", nargs=2, metavar=("USERNAME", "PASSWORD"), type=str, - help="the pair(s) of username and password for basic HTTP authentication" - ) - - return parser.parse_args(sys.argv[1:]) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/magneticow/magneticow/authorization.py b/magneticow/magneticow/authorization.py deleted file mode 100644 index ac9bf97..0000000 --- a/magneticow/magneticow/authorization.py +++ /dev/null @@ -1,61 +0,0 @@ -# magneticow - Lightweight web interface for magnetico. -# Copyright (C) 2017 Mert Bora ALPER -# Dedicated to Cemile Binay, in whose hands I thrived. -# -# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General -# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more -# details. -# -# You should have received a copy of the GNU Affero General Public License along with this program. If not, see -# . -import functools -import hashlib - -import flask - - -# Adapted from: http://flask.pocoo.org/snippets/8/ -# (c) Copyright 2010 - 2017 by Armin Ronacher -# BEGINNING OF THE 3RD PARTY COPYRIGHTED CONTENT -def is_authorized(supplied_username, supplied_password): - """ This function is called to check if a username / password combination is valid. """ - # Because we do monkey-patch! [in magneticow.__main__.py:main()] - app = flask.current_app - for username, password in app.arguments.user: # pylint: disable=maybe-no-member - if supplied_username == username and supplied_password == password: - return True - return False - - -def authenticate(): - """ Sends a 401 response that enables basic auth. """ - return flask.Response( - "Could not verify your access level for that URL.\n" - "You have to login with proper credentials", - 401, - {"WWW-Authenticate": 'Basic realm="Login Required"'} - ) - - -def requires_auth(f): - @functools.wraps(f) - def decorated(*args, **kwargs): - auth = flask.request.authorization - if not flask.current_app.arguments.noauth: - if not auth or not is_authorized(auth.username, auth.password): - return authenticate() - return f(*args, **kwargs) - return decorated -# END OF THE 3RD PARTY COPYRIGHTED CONTENT - - -def generate_feed_hash(username: str, password: str, filter_: str) -> str: - """ - Deterministically generates the feed hash from given username, password, and filter. - Hash is the hex encoding of the SHA256 sum. - """ - return hashlib.sha256((username + "\0" + password + "\0" + filter_).encode()).digest().hex() diff --git a/magneticow/magneticow/magneticow.py b/magneticow/magneticow/magneticow.py deleted file mode 100644 index b477d49..0000000 --- a/magneticow/magneticow/magneticow.py +++ /dev/null @@ -1,295 +0,0 @@ -# magneticow - Lightweight web interface for magnetico. -# Copyright (C) 2017 Mert Bora ALPER -# Dedicated to Cemile Binay, in whose hands I thrived. -# -# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General -# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more -# details. -# -# You should have received a copy of the GNU Affero General Public License along with this program. If not, see -# . -import collections -import datetime as dt -from datetime import datetime -import logging -import sqlite3 -import os - -import appdirs -import flask - -from magneticow import utils -from magneticow.authorization import requires_auth, generate_feed_hash - - -File = collections.namedtuple("file", ["path", "size"]) -Torrent = collections.namedtuple("torrent", ["info_hash", "name", "size", "discovered_on", "files"]) - - -app = flask.Flask(__name__) -app.config.from_object(__name__) - -# TODO: We should have been able to use flask.g but it does NOT persist across different requests so we resort back to -# this. Investigate the cause and fix it (I suspect of Gevent). -magneticod_db = None - -@app.route("/") -@requires_auth -def home_page(): - with magneticod_db: - # COUNT(ROWID) is much more inefficient since it scans the whole table, so use MAX(ROWID) - cur = magneticod_db.execute("SELECT MAX(ROWID) FROM torrents ;") - n_torrents = cur.fetchone()[0] or 0 - - return flask.render_template("homepage.html", n_torrents=n_torrents) - - -@app.route("/torrents/") -@requires_auth -def torrents(): - search = flask.request.args.get("search") - page = int(flask.request.args.get("page", 0)) - - context = { - "search": search, - "page": page - } - - SQL_query = """ - SELECT - info_hash, - name, - total_size, - discovered_on - FROM torrents - """ - if search: - SQL_query += """ - INNER JOIN ( - SELECT docid AS id, rank(matchinfo(fts_torrents, 'pcnxal')) AS rank - FROM fts_torrents - WHERE name MATCH ? - ) AS ranktable USING(id) - """ - SQL_query += """ - ORDER BY {} - LIMIT 20 OFFSET ? - """ - - sort_by = flask.request.args.get("sort_by") - allowed_sorts = [ - None, - "name ASC", - "name DESC", - "total_size ASC", - "total_size DESC", - "discovered_on ASC", - "discovered_on DESC" - ] - if sort_by not in allowed_sorts: - return flask.Response("Invalid value for `sort_by! (Allowed values are %s)" % (allowed_sorts, ), 400) - - if search: - if sort_by: - SQL_query = SQL_query.format(sort_by + ", " + "rank ASC") - else: - SQL_query = SQL_query.format("rank ASC") - else: - if sort_by: - SQL_query = SQL_query.format(sort_by + ", " + "id DESC") - else: - SQL_query = SQL_query.format("id DESC") - - with magneticod_db: - if search: - cur = magneticod_db.execute(SQL_query, (search, 20 * page)) - else: - cur = magneticod_db.execute(SQL_query, (20 * page, )) - context["torrents"] = [Torrent(t[0].hex(), t[1], utils.to_human_size(t[2]), - datetime.fromtimestamp(t[3]).strftime("%d/%m/%Y"), []) - for t in cur.fetchall()] - - if len(context["torrents"]) < 20: - context["next_page_exists"] = False - else: - context["next_page_exists"] = True - - if app.arguments.noauth: - context["subscription_url"] = "/feed/?filter%s" % search - else: - username, password = flask.request.authorization.username, flask.request.authorization.password - context["subscription_url"] = "/feed?filter=%s&hash=%s" % ( - search, generate_feed_hash(username, password, search)) - - if sort_by: - context["sorted_by"] = sort_by - - return flask.render_template("torrents.html", **context) - - -@app.route("/torrents//", defaults={"name": None}) -@requires_auth -def torrent_redirect(**kwargs): - try: - info_hash = bytes.fromhex(kwargs["info_hash"]) - assert len(info_hash) == 20 - except (AssertionError, ValueError): # In case info_hash variable is not a proper hex-encoded bytes - return flask.abort(400) - - with magneticod_db: - cur = magneticod_db.execute("SELECT name FROM torrents WHERE info_hash=? LIMIT 1;", (info_hash,)) - try: - name = cur.fetchone()[0] - except TypeError: # In case no results returned, TypeError will be raised when we try to subscript None object - return flask.abort(404) - - return flask.redirect("/torrents/%s/%s" % (kwargs["info_hash"], name), code=301) - - -@app.route("/torrents//") -@requires_auth -def torrent(**kwargs): - context = {} - - try: - info_hash = bytes.fromhex(kwargs["info_hash"]) - assert len(info_hash) == 20 - except (AssertionError, ValueError): # In case info_hash variable is not a proper hex-encoded bytes - return flask.abort(400) - - with magneticod_db: - cur = magneticod_db.execute("SELECT id, name, discovered_on FROM torrents WHERE info_hash=? LIMIT 1;", - (info_hash,)) - try: - torrent_id, name, discovered_on = cur.fetchone() - except TypeError: # In case no results returned, TypeError will be raised when we try to subscript None object - return flask.abort(404) - - cur = magneticod_db.execute("SELECT path, size FROM files WHERE torrent_id=?;", (torrent_id,)) - raw_files = cur.fetchall() - size = sum(f[1] for f in raw_files) - files = [File(f[0], utils.to_human_size(f[1])) for f in raw_files] - - context["torrent"] = Torrent(info_hash.hex(), name, utils.to_human_size(size), datetime.fromtimestamp(discovered_on).strftime("%d/%m/%Y"), files) - - return flask.render_template("torrent.html", **context) - - -@app.route("/statistics") -@requires_auth -def statistics(): - # Ahhh... - # Time is hard, really. magneticod used time.time() to save when a torrent is discovered, unaware that none of the - # specifications say anything about the timezones (or their irrelevance to the UNIX time) and about leap seconds in - # a year. - # Nevertheless, we still use it. In future, before v1.0.0, we may change it as we wish, offering a migration - # solution for the current users. But in the meanwhile, be aware that all your calculations will be a bit lousy, - # though within tolerable limits for a torrent search engine. - - with magneticod_db: - # latest_today is the latest UNIX timestamp of today, the very last second. - latest_today = int((dt.date.today() + dt.timedelta(days=1) - dt.timedelta(seconds=1)).strftime("%s")) - # Retrieve all the torrents discovered in the past 30 days (30 days * 24 hours * 60 minutes * 60 seconds...) - # Also, see http://www.sqlite.org/lang_datefunc.html for details of `date()`. - # Function Equivalent strftime() - # date(...) strftime('%Y-%m-%d', ...) - cur = magneticod_db.execute( - "SELECT date(discovered_on, 'unixepoch') AS day, count() FROM torrents WHERE discovered_on >= ? " - "GROUP BY day;", - (latest_today - 30 * 24 * 60 * 60, ) - ) - results = cur.fetchall() # for instance, [('2017-04-01', 17428), ('2017-04-02', 28342)] - - return flask.render_template("statistics.html", **{ - # We directly substitute them in the JavaScript code. - "dates": str([t[0] for t in results]), - "amounts": str([t[1] for t in results]) - }) - - -@app.route("/feed") -def feed(): - filter_ = flask.request.args["filter"] - # Check for all possible users who might be requesting. - # pylint disabled: because we do monkey-patch! [in magneticow.__main__.py:main()] - if not app.arguments.noauth: - hash_ = flask.request.args["hash"] - for username, password in app.arguments.user: # pylint: disable=maybe-no-member - if generate_feed_hash(username, password, filter_) == hash_: - break - else: - return flask.Response( - "Could not verify your access level for that URL (wrong hash).\n", - 401 - ) - - context = {} - - if filter_: - context["title"] = "`%s` - magneticow" % (filter_,) - with magneticod_db: - cur = magneticod_db.execute( - "SELECT " - " name, " - " info_hash " - "FROM torrents " - "INNER JOIN (" - " SELECT docid AS id, rank(matchinfo(fts_torrents, 'pcnxal')) AS rank " - " FROM fts_torrents " - " WHERE name MATCH ? " - " ORDER BY rank ASC" - " LIMIT 50" - ") AS ranktable USING(id);", - (filter_, ) - ) - context["items"] = [{"title": r[0], "info_hash": r[1].hex()} for r in cur] - else: - context["title"] = "The Newest Torrents - magneticow" - with magneticod_db: - cur = magneticod_db.execute( - "SELECT " - " name, " - " info_hash " - "FROM torrents " - "ORDER BY id DESC LIMIT 50" - ) - context["items"] = [{"title": r[0], "info_hash": r[1].hex()} for r in cur] - - return flask.render_template("feed.xml", **context), 200, {"Content-Type": "application/rss+xml; charset=utf-8"} - - -def initialize_magneticod_db() -> None: - global magneticod_db - - logging.info("Connecting to magneticod's database...") - - magneticod_db_path = os.path.join(appdirs.user_data_dir("magneticod"), "database.sqlite3") - magneticod_db = sqlite3.connect(magneticod_db_path, isolation_level=None) - - logging.info("Preparing for the full-text search (this might take a while)...") - with magneticod_db: - magneticod_db.execute("PRAGMA journal_mode=WAL;") - - magneticod_db.execute("CREATE INDEX IF NOT EXISTS discovered_on_index ON torrents (discovered_on);") - magneticod_db.execute("CREATE INDEX IF NOT EXISTS info_hash_index ON torrents (info_hash);") - magneticod_db.execute("CREATE INDEX IF NOT EXISTS file_info_hash_index ON files (torrent_id);") - - magneticod_db.execute("CREATE VIRTUAL TABLE temp.fts_torrents USING fts4(name);") - magneticod_db.execute("INSERT INTO fts_torrents (docid, name) SELECT id, name FROM torrents;") - magneticod_db.execute("INSERT INTO fts_torrents (fts_torrents) VALUES ('optimize');") - - magneticod_db.execute("CREATE TEMPORARY TRIGGER on_torrents_insert AFTER INSERT ON torrents FOR EACH ROW BEGIN" - " INSERT INTO fts_torrents (docid, name) VALUES (NEW.id, NEW.name);" - "END;") - - magneticod_db.create_function("rank", 1, utils.rank) - - -def close_db() -> None: - logging.info("Closing magneticod database...") - if magneticod_db is not None: - magneticod_db.close() diff --git a/magneticow/magneticow/utils.py b/magneticow/magneticow/utils.py deleted file mode 100644 index 5125b20..0000000 --- a/magneticow/magneticow/utils.py +++ /dev/null @@ -1,69 +0,0 @@ -# magneticow - Lightweight web interface for magnetico. -# Copyright (C) 2017 Mert Bora ALPER -# Dedicated to Cemile Binay, in whose hands I thrived. -# -# This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General -# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied -# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more -# details. -# -# You should have received a copy of the GNU Affero General Public License along with this program. If not, see -# . -from math import log10 -from struct import unpack_from - - -# Source: http://stackoverflow.com/a/1094933 -# (primarily: https://web.archive.org/web/20111010015624/http://blogmag.net/blog/read/38/Print_human_readable_file_size) -def to_human_size(num, suffix='B'): - for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: - if abs(num) < 1024: - return "%3.1f %s%s" % (num, unit, suffix) - num /= 1024 - return "%.1f %s%s" % (num, 'Yi', suffix) - - -def rank(blob): - # TODO: is there a way to futher optimize this? - p, c, n = unpack_from("=LLL", blob, 0) - - x = [] # list of tuples - for i in range(12, 12 + 3*c*p*4, 3*4): - x0, x1, x2 = unpack_from("=LLL", blob, i) - if x1 != 0: # skip if it's index column - x.append((x0, x1, x2)) - - # Ignore the first column (torrent_id) - avgdl = unpack_from("=L", blob, 12 + 3*c*p*4)[0] - - # Ignore the first column (torrent_id) - l = unpack_from("=L", blob, (12 + 3*c*p*4) + 4*c)[0] - - # Multiply by -1 so that sorting in the ASC order would yield the best match first - return -1 * okapi_bm25(term_frequencies=[X[0] for X in x], dl=l, avgdl=avgdl, N=n, nq=[X[2] for X in x]) - - -# TODO: check if I got it right =) -def okapi_bm25(term_frequencies, dl, avgdl, N, nq, k1=1.2, b=0.75): - """ - - :param term_frequencies: List of frequencies of each term in the document. - :param dl: Length of the document in words. - :param avgdl: Average document length in the collection. - :param N: Total number of documents in the collection. - :param nq: List of each numbers of documents containing term[i] for each term. - :param k1: Adjustable constant; = 1.2 in FTS5 extension of SQLite3. - :param b: Adjustable constant; = 0.75 in FTS5 extension of SQLite3. - :return: - """ - return sum( - log10((N - nq[i] + 0.5) / (nq[i] + 0.5)) * - ( - (term_frequencies[i] * (k1 + 1)) / - (term_frequencies[i] + k1 * (1 - b + b * dl / avgdl)) - ) - for i in range(len(term_frequencies)) - ) diff --git a/magneticow/main.go b/magneticow/main.go new file mode 100644 index 0000000..64fac03 --- /dev/null +++ b/magneticow/main.go @@ -0,0 +1,304 @@ +package main + +import ( + "html/template" + "log" + "net/http" + "os" + "strings" + + "github.com/dustin/go-humanize" + "github.com/gorilla/mux" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "encoding/hex" + "magnetico/persistence" + "strconv" + "time" + "unsafe" +) + +const N_TORRENTS = 20 + +var templates map[string]*template.Template +var database persistence.Database + +// ========= TD: TemplateData ========= +type HomepageTD struct { + Count uint +} + +type TorrentsTD struct { + Search string + SubscriptionURL string + Torrents []persistence.TorrentMetadata + Before int64 + After int64 + SortedBy string + NextPageExists bool +} + +type TorrentTD struct { +} + +type FeedTD struct { +} + +type StatisticsTD struct { +} + +func main() { + loggerLevel := zap.NewAtomicLevel() + // Logging levels: ("debug", "info", "warn", "error", "dpanic", "panic", and "fatal"). + logger := zap.New(zapcore.NewCore( + zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), + zapcore.Lock(os.Stderr), + loggerLevel, + )) + defer logger.Sync() + zap.ReplaceGlobals(logger) + + zap.L().Info("magneticow v0.7.0 has been started.") + zap.L().Info("Copyright (C) 2017 Mert Bora ALPER .") + zap.L().Info("Dedicated to Cemile Binay, in whose hands I thrived.") + + router := mux.NewRouter() + router.HandleFunc("/", rootHandler) + router.HandleFunc("/torrents", torrentsHandler) + router.HandleFunc("/torrents/{infohash:[a-z0-9]{40}}", torrentsInfohashHandler) + router.HandleFunc("/statistics", statisticsHandler) + router.PathPrefix("/static").HandlerFunc(staticHandler) + + router.HandleFunc("/feed", feedHandler) + + templateFunctions := template.FuncMap{ + "add": func(augend int, addends int) int { + return augend + addends + }, + + "subtract": func(minuend int, subtrahend int) int { + return minuend - subtrahend + }, + + "bytesToHex": func(bytes []byte) string { + return hex.EncodeToString(bytes) + }, + + "unixTimeToYearMonthDay": func(s int64) string { + tm := time.Unix(s, 0) + // > Format and Parse use example-based layouts. Usually you’ll use a constant from time + // > for these layouts, but you can also supply custom layouts. Layouts must use the + // > reference time Mon Jan 2 15:04:05 MST 2006 to show the pattern with which to + // > format/parse a given time/string. The example time must be exactly as shown: the + // > year 2006, 15 for the hour, Monday for the day of the week, etc. + // https://gobyexample.com/time-formatting-parsing + // Why you gotta be so weird Go? + return tm.Format("02/01/2006") + }, + + "humanizeSize": func(s uint64) string { + return humanize.IBytes(s) + }, + } + + templates = make(map[string]*template.Template) + templates["feed"] = template.Must(template.New("feed").Parse(string(mustAsset("templates/feed.xml")))) + templates["homepage"] = template.Must(template.New("homepage").Parse(string(mustAsset("templates/homepage.html")))) + templates["statistics"] = template.Must(template.New("statistics").Parse(string(mustAsset("templates/statistics.html")))) + templates["torrent"] = template.Must(template.New("torrent").Funcs(templateFunctions).Parse(string(mustAsset("templates/torrent.html")))) + templates["torrents"] = template.Must(template.New("torrents").Funcs(templateFunctions).Parse(string(mustAsset("templates/torrents.html")))) + + var err error + database, err = persistence.MakeDatabase("sqlite3:///home/bora/.local/share/magneticod/database.sqlite3", unsafe.Pointer(logger)) + if err != nil { + panic(err.Error()) + } + + zap.L().Info("magneticow is ready to serve!") + http.ListenAndServe(":8080", router) +} + +// DONE +func rootHandler(w http.ResponseWriter, r *http.Request) { + count, err := database.GetNumberOfTorrents() + if err != nil { + panic(err.Error()) + } + templates["homepage"].Execute(w, HomepageTD{ + Count: count, + }) +} + +func torrentsHandler(w http.ResponseWriter, r *http.Request) { + queryValues := r.URL.Query() + + // Parses `before` and `after` parameters in the URL query following the conditions below: + // * `before` and `after` cannot be both supplied at the same time. + // * `before` -if supplied- cannot be less than or equal to zero. + // * `after` -if supplied- cannot be greater than the current Unix time. + // * if `before` is not supplied, it is set to the current Unix time. + qBefore, qAfter := (int64)(-1), (int64)(-1) + var err error + if queryValues.Get("before") != "" { + qBefore, err = strconv.ParseInt(queryValues.Get("before"), 10, 64) + if err != nil { + panic(err.Error()) + } + if qBefore <= 0 { + panic("before parameter is less than or equal to zero!") + } + } else if queryValues.Get("after") != "" { + if qBefore != -1 { + panic("both before and after supplied") + } + qAfter, err = strconv.ParseInt(queryValues.Get("after"), 10, 64) + if err != nil { + panic(err.Error()) + } + if qAfter > time.Now().Unix() { + panic("after parameter is greater than the current Unix time!") + } + } else { + qBefore = time.Now().Unix() + } + + var torrents []persistence.TorrentMetadata + if qBefore != -1 { + torrents, err = database.GetNewestTorrents(N_TORRENTS, qBefore) + } else { + torrents, err = database.QueryTorrents( + queryValues.Get("search"), + persistence.BY_DISCOVERED_ON, + true, + false, + N_TORRENTS, + qAfter, + true, + ) + } + if err != nil { + panic(err.Error()) + } + + // TODO: for testing, REMOVE + torrents[2].HasReadme = true + + templates["torrents"].Execute(w, TorrentsTD{ + Search: "", + SubscriptionURL: "borabora", + Torrents: torrents, + Before: torrents[len(torrents)-1].DiscoveredOn, + After: torrents[0].DiscoveredOn, + SortedBy: "anan", + NextPageExists: true, + }) + +} + +func newestTorrentsHandler(w http.ResponseWriter, r *http.Request) { + queryValues := r.URL.Query() + + qBefore, qAfter := (int64)(-1), (int64)(-1) + var err error + if queryValues.Get("before") != "" { + qBefore, err = strconv.ParseInt(queryValues.Get("before"), 10, 64) + if err != nil { + panic(err.Error()) + } + } else if queryValues.Get("after") != "" { + if qBefore != -1 { + panic("both before and after supplied") + } + qAfter, err = strconv.ParseInt(queryValues.Get("after"), 10, 64) + if err != nil { + panic(err.Error()) + } + } else { + qBefore = time.Now().Unix() + } + + var torrents []persistence.TorrentMetadata + if qBefore != -1 { + torrents, err = database.QueryTorrents( + "", + persistence.BY_DISCOVERED_ON, + true, + false, + N_TORRENTS, + qBefore, + false, + ) + } else { + torrents, err = database.QueryTorrents( + "", + persistence.BY_DISCOVERED_ON, + false, + false, + N_TORRENTS, + qAfter, + true, + ) + } + if err != nil { + panic(err.Error()) + } + + templates["torrents"].Execute(w, TorrentsTD{ + Search: "", + SubscriptionURL: "borabora", + Torrents: torrents, + Before: torrents[len(torrents)-1].DiscoveredOn, + After: torrents[0].DiscoveredOn, + SortedBy: "anan", + NextPageExists: true, + }) +} + +func torrentsInfohashHandler(w http.ResponseWriter, r *http.Request) { + // show torrents/{infohash} + infoHash, err := hex.DecodeString(mux.Vars(r)["infohash"]) + if err != nil { + panic(err.Error()) + } + + torrent, err := database.GetTorrent(infoHash) + if err != nil { + panic(err.Error()) + } + + templates["torrent"].Execute(w, torrent) +} + +func statisticsHandler(w http.ResponseWriter, r *http.Request) { + +} + +func feedHandler(w http.ResponseWriter, r *http.Request) { + +} + +func staticHandler(w http.ResponseWriter, r *http.Request) { + data, err := Asset(r.URL.Path[1:]) + if err != nil { + http.NotFound(w, r) + return + } + + var contentType string + if strings.HasSuffix(r.URL.Path, ".css") { + contentType = "text/css; charset=utf-8" + } else { // fallback option + contentType = http.DetectContentType(data) + } + w.Header().Set("Content-Type", contentType) + w.Write(data) +} + +func mustAsset(name string) []byte { + data, err := Asset(name) + if err != nil { + log.Panicf("Could NOT access the requested resource `%s`: %s (please inform us, this is a BUG!)", name, err.Error()) + } + return data +} diff --git a/magneticow/setup.py b/magneticow/setup.py deleted file mode 100644 index 6258d7d..0000000 --- a/magneticow/setup.py +++ /dev/null @@ -1,41 +0,0 @@ -from setuptools import setup - - -def read_file(path): - with open(path) as file: - return file.read() - - -setup( - name="magneticow", - version="0.6.0", - description="Lightweight web interface for magnetico.", - long_description=read_file("README.rst"), - url="http://magnetico.org", - author="Mert Bora ALPER", - author_email="bora@boramalper.org", - license="GNU Affero General Public License v3 or later (AGPLv3+)", - packages=["magneticow"], - include_package_data=True, - zip_safe=False, - entry_points={ - "console_scripts": ["magneticow=magneticow.__main__:main"] - }, - - install_requires=[ - "appdirs >= 1.4.3", - "flask >= 0.12.1", - "gevent >= 1.2.1" - ], - - classifiers=[ - "Development Status :: 2 - Pre-Alpha", - "Environment :: Web Environment", - "Intended Audience :: End Users/Desktop", - "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", - "Natural Language :: English", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: Implementation :: CPython" - ] -) diff --git a/magneticow/systemd/magneticow.service b/magneticow/systemd/magneticow.service deleted file mode 100644 index f0870a8..0000000 --- a/magneticow/systemd/magneticow.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=magneticow: lightweight web interface for magnetico - -[Service] -ExecStart=~/.local/bin/magneticow --port PORT --user USERNAME PASSWORD -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/persistence/interface.go b/persistence/interface.go new file mode 100644 index 0000000..740c6a7 --- /dev/null +++ b/persistence/interface.go @@ -0,0 +1,117 @@ +package persistence + +import ( + "fmt" + "go.uber.org/zap" + "net/url" +) + +type Database interface { + Engine() databaseEngine + DoesTorrentExist(infoHash []byte) (bool, error) + AddNewTorrent(infoHash []byte, name string, files []File) error + Close() error + + // GetNumberOfTorrents returns the number of torrents saved in the database. Might be an + // approximation. + GetNumberOfTorrents() (uint, error) + // QueryTorrents returns @n torrents + // * that are discovered before the @timePoint if @isAfter is false, else that are + // discovered after the @timePoint, + // * that match the @query if it's not empty, + // ordered by the @orderBy in ascending order if @isDescending is false, else in descending + // order. + QueryTorrents(query string, orderBy orderingCriteria, ord order, n uint, when presence, timePoint int64) ([]TorrentMetadata, error) + // GetTorrents returns the TorrentExtMetadata for the torrent of the given InfoHash. Might return + // nil, nil if the torrent does not exist in the database. + GetTorrent(infoHash []byte) (*TorrentMetadata, error) + GetFiles(infoHash []byte) ([]File, error) + GetStatistics(from ISO8601, period uint) (*Statistics, error) +} + +type orderingCriteria uint8 + +const ( + BY_RELEVANCE orderingCriteria = 1 + BY_SIZE = 2 + BY_DISCOVERED_ON = 3 + BY_N_FILES = 4 +) + +type order uint8 + +const ( + ASCENDING order = 1 + DESCENDING = 2 +) + +type presence uint8 + +const ( + BEFORE presence = 1 + AFTER = 2 +) + +type statisticsGranularity uint8 +type ISO8601 string + +const ( + HOURLY_STATISTICS statisticsGranularity = 1 + DAILY_STATISTICS = 2 + WEEKLY_STATISTICS = 3 + MONTHLY_STATISTICS = 4 + YEARLY_STATISTICS = 5 +) + +type databaseEngine uint8 + +const ( + SQLITE3_ENGINE databaseEngine = 1 +) + +type Statistics struct { + Granularity statisticsGranularity + From ISO8601 + Period uint + + // All these slices below have the exact length equal to the Period. + NTorrentsDiscovered []uint + NFilesDiscovered []uint +} + +type File struct { + Size int64 + Path string +} + +type TorrentMetadata struct { + InfoHash []byte + Name string + Size uint64 + DiscoveredOn int64 + NFiles uint +} + +func MakeDatabase(rawURL string, enableFTS bool, logger *zap.Logger) (Database, error) { + if logger != nil { + zap.ReplaceGlobals(logger) + } + + url_, err := url.Parse(rawURL) + if err != nil { + return nil, err + } + + switch url_.Scheme { + case "sqlite3": + return makeSqlite3Database(url_, enableFTS) + + case "postgresql": + return nil, fmt.Errorf("postgresql is not yet supported!") + + case "mysql": + return nil, fmt.Errorf("mysql is not yet supported!") + } + + return nil, fmt.Errorf("unknown URI scheme (database engine)!") +} diff --git a/src/persistence/sqlite3.go b/persistence/sqlite3.go similarity index 64% rename from src/persistence/sqlite3.go rename to persistence/sqlite3.go index 6fed455..d43583d 100644 --- a/src/persistence/sqlite3.go +++ b/persistence/sqlite3.go @@ -1,15 +1,16 @@ package persistence import ( - "net/url" - "path" - "os" - "fmt" "database/sql" - "regexp" - - "go.uber.org/zap" + "fmt" + "net/url" + "os" + "path" "time" + + _ "github.com/mattn/go-sqlite3" + "go.uber.org/zap" + "math" ) type sqlite3Database struct { @@ -20,7 +21,7 @@ func (db *sqlite3Database) Engine() databaseEngine { return SQLITE3_ENGINE } -func makeSqlite3Database(url_ *url.URL) (Database, error) { +func makeSqlite3Database(url_ *url.URL, enableFTS bool) (Database, error) { db := new(sqlite3Database) dbDir, _ := path.Split(url_.Path) @@ -31,18 +32,18 @@ func makeSqlite3Database(url_ *url.URL) (Database, error) { var err error db.conn, err = sql.Open("sqlite3", url_.Path) if err != nil { - return nil, err + return nil, fmt.Errorf("sql.Open: %s", err.Error()) } // > Open may just validate its arguments without creating a connection to the database. To - // > verify that the data source name is valid, call Ping. + // > verify that the data source Name is valid, call Ping. // https://golang.org/pkg/database/sql/#Open if err = db.conn.Ping(); err != nil { - return nil, err + return nil, fmt.Errorf("sql.DB.Ping: %s", err.Error()) } if err := db.setupDatabase(); err != nil { - return nil, err + return nil, fmt.Errorf("setupDatabase: %s", err.Error()) } return db, nil @@ -51,7 +52,7 @@ func makeSqlite3Database(url_ *url.URL) (Database, error) { func (db *sqlite3Database) DoesTorrentExist(infoHash []byte) (bool, error) { rows, err := db.conn.Query("SELECT 1 FROM torrents WHERE info_hash = ?;", infoHash) if err != nil { - return false, err; + return false, err } // If rows.Next() returns true, meaning that the torrent is in the database, return true; else @@ -59,37 +60,10 @@ func (db *sqlite3Database) DoesTorrentExist(infoHash []byte) (bool, error) { exists := rows.Next() if err = rows.Close(); err != nil { - return false, err; + return false, err } - return exists, nil; -} - -func (db *sqlite3Database) GiveAnIncompleteTorrent(pathRegex *regexp.Regexp, maxSize uint) (infoHash []byte, path string, err error) { - rows, err := db.conn.Query("SELECT info_hash FROM torrents WHERE has_readme = 0;") - if err != nil { - return nil, "", err - } - - if rows.Next() != true { - return nil, "", nil - } - - if err = rows.Scan(&infoHash); err != nil { - return nil, "", err - } - - if err = rows.Close(); err != nil { - return nil, "", err - } - - // TODO - return infoHash, "", nil -} - -func (db *sqlite3Database) GiveAStaleTorrent() (infoHash []byte, err error) { - // TODO - return nil, nil + return exists, nil } func (db *sqlite3Database) AddNewTorrent(infoHash []byte, name string, files []File) error { @@ -103,9 +77,9 @@ func (db *sqlite3Database) AddNewTorrent(infoHash []byte, name string, files []F // add it. exists, err := db.DoesTorrentExist(infoHash) if err != nil { - return err; + return err } else if exists { - return nil; + return nil } tx, err := db.conn.Begin() @@ -123,15 +97,19 @@ func (db *sqlite3Database) AddNewTorrent(infoHash []byte, name string, files []F total_size += file.Size } + // This is a workaround for a bug: the database will not accept total_size to be zero. + if total_size == 0 { + return nil + } + res, err := tx.Exec(` INSERT INTO torrents ( info_hash, name, total_size, - discovered_on, - n_files, - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?); - `, infoHash, name, total_size, time.Now().Unix(), len(files)) + discovered_on + ) VALUES (?, ?, ?, ?); + `, infoHash, name, total_size, time.Now().Unix()) if err != nil { return err } @@ -142,7 +120,7 @@ func (db *sqlite3Database) AddNewTorrent(infoHash []byte, name string, files []F } for _, file := range files { - _, err = tx.Exec("INSERT INTO files (torrent_id, size, path) VALUES (?, ?, ?);", + _, err = tx.Exec("INSERT INTO files (torrent_id, Size, path) VALUES (?, ?, ?);", lastInsertId, file.Size, file.Path, ) if err != nil { @@ -158,25 +136,13 @@ func (db *sqlite3Database) AddNewTorrent(infoHash []byte, name string, files []F return nil } -func (db *sqlite3Database) AddReadme(infoHash []byte, path string, content string) error { - _, err := db.conn.Exec( - `UPDATE files SET is_readme = 1, content = ? - WHERE path = ? AND (SELECT id FROM torrents WHERE info_hash = ?) = torrent_id;`, - content, path, infoHash, - ) - if err != nil { - return err - } - - return nil -} func (db *sqlite3Database) Close() error { return db.conn.Close() } func (db *sqlite3Database) GetNumberOfTorrents() (uint, error) { - // COUNT(ROWID) is much more inefficient since it scans the whole table, so use MAX(ROWID) + // COUNT(1) is much more inefficient since it scans the whole table, so use MAX(ROWID) rows, err := db.conn.Query("SELECT MAX(ROWID) FROM torrents;") if err != nil { return 0, err @@ -198,45 +164,21 @@ func (db *sqlite3Database) GetNumberOfTorrents() (uint, error) { return n, nil } -func (db *sqlite3Database) NewestTorrents(n uint) ([]TorrentMetadata, error) { - rows, err := db.conn.Query(` - SELECT - info_hash, - name, - total_size, - discovered_on, - has_readme, - n_files, - n_seeders, - n_leechers, - updated_on - FROM torrents - ORDER BY discovered_on DESC LIMIT ?; - `, n, - ) - if err != nil { - return nil, err +func (db *sqlite3Database) QueryTorrents(query string, orderBy orderingCriteria, ord order, n uint, when presence, timePoint int64) ([]TorrentMetadata, error) { + if query == "" && orderBy == BY_RELEVANCE { + return nil, fmt.Errorf("torrents cannot be ordered by \"relevance\" when the query is empty") } - var torrents []TorrentMetadata - for rows.Next() { - tm := new(TorrentMetadata) - rows.Scan( - &tm.infoHash, &tm.name, &tm.discoveredOn, &tm.hasReadme, &tm.nFiles, &tm.nSeeders, - &tm.nLeechers, &tm.updatedOn, - ) - torrents = append(torrents, *tm) + if timePoint == 0 && when == BEFORE { + return nil, fmt.Errorf("nothing can come \"before\" time 0") } - if err = rows.Close(); err != nil { - return nil, err + if timePoint == math.MaxInt64 && when == AFTER { + return nil, fmt.Errorf("nothing can come \"after\" time %d", math.MaxInt64) } - return torrents, nil -} + // TODO -func (db *sqlite3Database) SearchTorrents(query string, orderBy orderingCriteria, descending bool, mustHaveReadme bool) ([]TorrentMetadata, error) { // TODO - // TODO: return nil, nil } @@ -245,13 +187,9 @@ func (db *sqlite3Database) GetTorrent(infoHash []byte) (*TorrentMetadata, error) `SELECT info_hash, name, - size, + total_size, discovered_on, - has_readme, - n_files, - n_seeders, - n_leechers, - updated_on + (SELECT COUNT(1) FROM files WHERE torrent_id = torrents.id) AS n_files FROM torrents WHERE info_hash = ?`, infoHash, @@ -261,32 +199,34 @@ func (db *sqlite3Database) GetTorrent(infoHash []byte) (*TorrentMetadata, error) } if rows.Next() != true { + zap.L().Warn("torrent not found amk") return nil, nil } - tm := new(TorrentMetadata) - rows.Scan( - &tm.infoHash, &tm.name, &tm.discoveredOn, &tm.hasReadme, &tm.nFiles, &tm.nSeeders, - &tm.nLeechers, &tm.updatedOn, - ) - + var tm TorrentMetadata + rows.Scan(&tm.InfoHash, &tm.Name, &tm.Size, &tm.DiscoveredOn, &tm.NFiles) if err = rows.Close(); err != nil { return nil, err } - return tm, nil + return &tm, nil } func (db *sqlite3Database) GetFiles(infoHash []byte) ([]File, error) { - // TODO - return nil, nil -} + rows, err := db.conn.Query("SELECT size, path FROM files WHERE torrent_id = ?;", infoHash) + if err != nil { + return nil, err + } -func (db *sqlite3Database) GetReadme(infoHash []byte) (string, error) { - // TODO - return "", nil -} + var files []File + for rows.Next() { + var file File + rows.Scan(&file.Size, &file.Path) + files = append(files, file) + } + return files, nil +} func (db *sqlite3Database) GetStatistics(from ISO8601, period uint) (*Statistics, error) { // TODO @@ -294,6 +234,7 @@ func (db *sqlite3Database) GetStatistics(from ISO8601, period uint) (*Statistics } func (db *sqlite3Database) commitQueuedTorrents() error { + // TODO return nil } @@ -317,14 +258,15 @@ func (db *sqlite3Database) setupDatabase() error { PRAGMA journal_mode=WAL; PRAGMA temp_store=1; PRAGMA foreign_keys=ON; + PRAGMA encoding="UTF-8"; `) if err != nil { - return err + return fmt.Errorf("sql.DB.Exec (PRAGMAs): %s", err.Error()) } tx, err := db.conn.Begin() if err != nil { - return err + return fmt.Errorf("sql.DB.Begin: %s", err.Error()) } // If everything goes as planned and no error occurs, we will commit the transaction before // returning from the function so the tx.Rollback() call will fail, trying to rollback a @@ -350,46 +292,54 @@ func (db *sqlite3Database) setupDatabase() error { ); `) if err != nil { - return err + return fmt.Errorf("sql.Tx.Exec (v0): %s", err.Error()) } // Get the user_version: - res, err := tx.Query("PRAGMA user_version;") + rows, err := tx.Query("PRAGMA user_version;") if err != nil { - return err + return fmt.Errorf("sql.Tx.Query (user_version): %s", err.Error()) } - var userVersion int; - if res.Next() != true { - return fmt.Errorf("PRAGMA user_version did not return any rows!") + var userVersion int + if rows.Next() != true { + return fmt.Errorf("sql.Rows.Next (user_version): PRAGMA user_version did not return any rows!") } - if err = res.Scan(&userVersion); err != nil { - return err + if err = rows.Scan(&userVersion); err != nil { + return fmt.Errorf("sql.Rows.Scan (user_version): %s", err.Error()) + } + // Close your rows lest you get "database table is locked" error(s)! + // See https://github.com/mattn/go-sqlite3/issues/2741 + if err = rows.Close(); err != nil { + return fmt.Errorf("sql.Rows.Close (user_version): %s", err.Error()) } switch userVersion { - // Upgrade from user_version 0 to 1 - // The Change: - // * `info_hash_index` is recreated as UNIQUE. case 0: - zap.S().Warnf("Updating database schema from 0 to 1... (this might take a while)") + // Upgrade from user_version 0 to 1 + // Changes: + // * `info_hash_index` is recreated as UNIQUE. + zap.L().Warn("Updating database schema from 0 to 1... (this might take a while)") _, err = tx.Exec(` DROP INDEX info_hash_index; CREATE UNIQUE INDEX info_hash_index ON torrents (info_hash); PRAGMA user_version = 1; `) if err != nil { - return err + return fmt.Errorf("sql.Tx.Exec (v0 -> v1): %s", err.Error()) } fallthrough - // Upgrade from user_version 1 to 2 - // The Change: - // * Added `is_readme` and `content` columns to the `files` table, and the constraints & the - // the indices they entail. - // * Added unique index `readme_index` on `files` table. + case 1: - zap.S().Warnf("Updating database schema from 1 to 2... (this might take a while)") - // We introduce two new columns here: content BLOB, and is_readme INTEGER which we treat as - // a bool (hence the CHECK). + // Upgrade from user_version 1 to 2 + // Changes: + // * Added `n_seeders`, `n_leechers`, and `updated_on` columns to the `torrents` table, and + // the constraints they entail. + // * Added `is_readme` and `content` columns to the `files` table, and the constraints & the + // the indices they entail. + // * Added unique index `readme_index` on `files` table. + zap.L().Warn("Updating database schema from 1 to 2... (this might take a while)") + // We introduce two new columns in `files`: content BLOB, and is_readme INTEGER which we + // treat as a bool (NULL for false, and 1 for true; see the CHECK statement). // The reason for the change is that as we introduce the new "readme" feature which // downloads a readme file as a torrent descriptor, we needed to store it somewhere in the // database with the following conditions: @@ -402,26 +352,31 @@ func (db *sqlite3Database) setupDatabase() error { // // Regarding the implementation details, following constraints arise: // - // 1. The column is_readme is either NULL or 1, and if it is 1, then content column cannot - // be NULL (but might be an empty BLOB). Vice versa, if content column of a row is, - // NULL then is_readme must be NULL. + // 1. The column is_readme is either NULL or 1, and if it is 1, then column content cannot + // be NULL (but might be an empty BLOB). Vice versa, if column content of a row is, + // NULL then column is_readme must be NULL. // // This is to prevent unused content fields filling up the database, and to catch // programmers' errors. _, err = tx.Exec(` + ALTER TABLE torrents ADD COLUMN updated_on INTEGER CHECK (updated_on > 0) DEFAULT NULL; + ALTER TABLE torrents ADD COLUMN n_seeders INTEGER CHECK ((updated_on IS NOT NULL AND n_seeders >= 0) OR (updated_on IS NULL AND n_seeders IS NULL)) DEFAULT NULL; + ALTER TABLE torrents ADD COLUMN n_leechers INTEGER CHECK ((updated_on IS NOT NULL AND n_leechers >= 0) OR (updated_on IS NULL AND n_leechers IS NULL)) DEFAULT NULL; + ALTER TABLE files ADD COLUMN is_readme INTEGER CHECK (is_readme IS NULL OR is_readme=1) DEFAULT NULL; - ALTER TABLE files ADD COLUMN content BLOB CHECK((content IS NULL AND is_readme IS NULL) OR (content IS NOT NULL AND is_readme=1)) DEFAULT NULL; + ALTER TABLE files ADD COLUMN content TEXT CHECK ((content IS NULL AND is_readme IS NULL) OR (content IS NOT NULL AND is_readme=1)) DEFAULT NULL; CREATE UNIQUE INDEX readme_index ON files (torrent_id, is_readme); + PRAGMA user_version = 2; `) if err != nil { - return err + return fmt.Errorf("sql.Tx.Exec (v1 -> v2): %s", err.Error()) } } if err = tx.Commit(); err != nil { - return err + return fmt.Errorf("sql.Tx.Commit: %s", err.Error()) } return nil -} \ No newline at end of file +} diff --git a/pkg/.gitignore b/pkg/.gitignore deleted file mode 100644 index 5e7d273..0000000 --- a/pkg/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/pylintrc b/pylintrc deleted file mode 100644 index d105412..0000000 --- a/pylintrc +++ /dev/null @@ -1,407 +0,0 @@ -[MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=.git - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Pickle collected data for later comparisons. -persistent=no - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Use multiple processes to speed up Pylint. -jobs=4 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. This option is deprecated -# and it will be removed in Pylint 2.0. -optimize-ast=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence=INFERENCE - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=range-builtin-not-iterating,coerce-builtin,old-ne-operator,reduce-builtin,suppressed-message,parameter-unpacking,unichr-builtin,round-builtin,hex-method,dict-iter-method,basestring-builtin,no-absolute-import,using-cmp-argument,buffer-builtin,raw_input-builtin,delslice-method,filter-builtin-not-iterating,setslice-method,nonzero-method,import-star-module-level,useless-suppression,map-builtin-not-iterating,raising-string,file-builtin,dict-view-method,standarderror-builtin,long-suffix,print-statement,xrange-builtin,intern-builtin,input-builtin,metaclass-assignment,cmp-method,unpacking-in-except,cmp-builtin,next-method-called,coerce-method,apply-builtin,long-builtin,getslice-method,zip-builtin-not-iterating,backtick,execfile-builtin,unicode-builtin,old-division,indexing-exception,old-raise-syntax,oct-method,reload-builtin,old-octal-literal - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=colorized - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". This option is deprecated -# and it will be removed in Pylint 2.0. -files-output=no - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=120 - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - - -[BASIC] - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=1000 - - -[ELIF] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/src/magneticod/bittorrent/operations.go b/src/magneticod/bittorrent/operations.go deleted file mode 100644 index 845658c..0000000 --- a/src/magneticod/bittorrent/operations.go +++ /dev/null @@ -1,185 +0,0 @@ -package bittorrent - -import ( - "time" - "strings" - - "github.com/anacrolix/missinggo" - "github.com/anacrolix/torrent" - "github.com/anacrolix/torrent/metainfo" - "go.uber.org/zap" - "os" - "path" - "persistence" -) - - -func (ms *MetadataSink) awaitMetadata(infoHash metainfo.Hash, peer torrent.Peer) { - t, isNew := ms.client.AddTorrentInfoHash(infoHash) - // If the infoHash we added was not new (i.e. it's already being downloaded by the client) - // then t is the handle of the (old) torrent. We add the (presumably new) peer to the torrent - // so we can increase the chance of operation being successful, or that the metadata might be - // fetched. - t.AddPeers([]torrent.Peer{peer}) - if !isNew { - // Return immediately if we are trying to await on an ongoing metadata-fetching operation. - // Each ongoing operation should have one and only one "await*" function waiting on it. - return - } - - // Wait for the torrent client to receive the metadata for the torrent, meanwhile allowing - // termination to be handled gracefully. - var info *metainfo.Info - select { - case <- t.GotInfo(): - info = t.Info() - t.Drop() - - case <-time.After(5 * time.Minute): - zap.L().Sugar().Debugf("Fetcher timeout! %x", infoHash) - return - - case <- ms.termination: - return - } - - var files []persistence.File - for _, file := range info.Files { - files = append(files, persistence.File{ - Size: file.Length, - Path: file.DisplayPath(info), - }) - } - - var totalSize uint64 - for _, file := range files { - if file.Size < 0 { - // All files' sizes must be greater than or equal to zero, otherwise treat them as - // illegal and ignore. - zap.L().Sugar().Debugf("!!!! file size zero or less! \"%s\" (%d)", file.Path, file.Size) - return - } - totalSize += uint64(file.Size) - } - - ms.flush(Metadata{ - InfoHash: infoHash[:], - Name: info.Name, - TotalSize: totalSize, - DiscoveredOn: time.Now().Unix(), - Files: files, - Peers: nil, - }) -} - - -func (fs *FileSink) awaitFile(req *FileRequest) { - // Remove the download directory of the torrent after the operation is completed. - // TODO: what if RemoveAll() returns error, do we care, and if we do, how to handle it? - defer os.RemoveAll(path.Join(fs.baseDownloadDir, string(req.InfoHash))) - - var infoHash_ [20]byte - copy(infoHash_[:], req.InfoHash) - t, isNew := fs.client.AddTorrentInfoHash(infoHash_) - if len(req.Peers) > 0 { - t.AddPeers(req.Peers) - } - if !isNew { - // Return immediately if we are trying to await on an ongoing file-downloading operation. - // Each ongoing operation should have one and only one "await*" function waiting on it. - return - } - - // Setup & start the timeout timer. - timeout := time.After(fs.timeoutDuration) - - // Once we return from this function, drop the torrent from the client. - // TODO: Check if dropping a torrent also cancels any outstanding read operations? - defer t.Drop() - - select { - case <-t.GotInfo(): - - case <- timeout: - return - } - - var match *torrent.File - for _, file := range t.Files() { - if file.Path() == req.Path { - match = &file - } else { - file.Cancel() - } - } - if match == nil { - var filePaths []string - for _, file := range t.Files() { filePaths = append(filePaths, file.Path()) } - - zap.L().Warn( - "The leech (FileSink) has been requested to download a file which does not exist!", - zap.ByteString("torrent", req.InfoHash), - zap.String("requestedFile", req.Path), - zap.Strings("allFiles", filePaths), - ) - } - - reader := t.NewReader() - defer reader.Close() - - fileDataChan := make(chan []byte) - go downloadFile(*match, reader, fileDataChan) - - select { - case fileData := <-fileDataChan: - if fileData != nil { - fs.flush(FileResult{ - Request: req, - FileData: fileData, - }) - } - - case <- timeout: - zap.L().Debug( - "Timeout while downloading a file!", - zap.ByteString("torrent", req.InfoHash), - zap.String("file", req.Path), - ) - } -} - - -func downloadFile(file torrent.File, reader *torrent.Reader, fileDataChan chan<- []byte) { - readSeeker := missinggo.NewSectionReadSeeker(reader, file.Offset(), file.Length()) - - fileData := make([]byte, file.Length()) - n, err := readSeeker.Read(fileData) - if int64(n) != file.Length() { - infoHash := file.Torrent().InfoHash() - zap.L().Debug( - "Not all of a file could be read!", - zap.ByteString("torrent", infoHash[:]), - zap.String("file", file.Path()), - zap.Int64("fileLength", file.Length()), - zap.Int("n", n), - ) - fileDataChan <- nil - return - } - if err != nil { - infoHash := file.Torrent().InfoHash() - zap.L().Debug( - "Error while downloading a file!", - zap.Error(err), - zap.ByteString("torrent", infoHash[:]), - zap.String("file", file.Path()), - zap.Int64("fileLength", file.Length()), - zap.Int("n", n), - ) - fileDataChan <- nil - return - } - - fileDataChan <- fileData -} - diff --git a/src/magneticod/bittorrent/sinkFile.go b/src/magneticod/bittorrent/sinkFile.go deleted file mode 100644 index 5d75c0c..0000000 --- a/src/magneticod/bittorrent/sinkFile.go +++ /dev/null @@ -1,117 +0,0 @@ -package bittorrent - -import ( - "net" - "path" - "time" - - "github.com/anacrolix/dht" - "github.com/anacrolix/torrent" - "github.com/anacrolix/torrent/storage" - "github.com/Wessie/appdirs" - "go.uber.org/zap" -) - -type FileRequest struct { - InfoHash []byte - Path string - Peers []torrent.Peer -} - -type FileResult struct { - // Request field is the original Request - Request *FileRequest - FileData []byte -} - -type FileSink struct { - baseDownloadDir string - client *torrent.Client - drain chan FileResult - terminated bool - termination chan interface{} - - timeoutDuration time.Duration -} - -// NewFileSink creates a new FileSink. -// -// cAddr : client address -// mlAddr: mainline DHT node address -func NewFileSink(cAddr, mlAddr string, timeoutDuration time.Duration) *FileSink { - fs := new(FileSink) - - mlUDPAddr, err := net.ResolveUDPAddr("udp", mlAddr) - if err != nil { - zap.L().Fatal("Could NOT resolve UDP addr!", zap.Error(err)) - return nil - } - - // Make sure to close the mlUDPConn before returning from this function in case of an error. - mlUDPConn, err := net.ListenUDP("udp", mlUDPAddr) - if err != nil { - zap.L().Fatal("Could NOT listen UDP (file sink)!", zap.Error(err)) - return nil - } - - fs.baseDownloadDir = path.Join( - appdirs.UserCacheDir("magneticod", "", "", true), - "downloads", - ) - - fs.client, err = torrent.NewClient(&torrent.Config{ - ListenAddr: cAddr, - DisableTrackers: true, - DHTConfig: dht.ServerConfig{ - Conn: mlUDPConn, - Passive: true, - NoSecurity: true, - }, - DefaultStorage: storage.NewFileByInfoHash(fs.baseDownloadDir), - }) - if err != nil { - zap.L().Fatal("Leech could NOT create a new torrent client!", zap.Error(err)) - mlUDPConn.Close() - return nil - } - - fs.drain = make(chan FileResult) - fs.termination = make(chan interface{}) - fs.timeoutDuration = timeoutDuration - - return fs -} - -// peer field is optional and might be nil. -func (fs *FileSink) Sink(infoHash []byte, path string, peers []torrent.Peer) { - if fs.terminated { - zap.L().Panic("Trying to Sink() an already closed FileSink!") - } - go fs.awaitFile(&FileRequest{ - InfoHash: infoHash, - Path: path, - Peers: peers, - }) -} - -func (fs *FileSink) Drain() <-chan FileResult { - if fs.terminated { - zap.L().Panic("Trying to Drain() an already closed FileSink!") - } - return fs.drain -} - - -func (fs *FileSink) Terminate() { - fs.terminated = true - close(fs.termination) - fs.client.Close() - close(fs.drain) -} - - -func (fs *FileSink) flush(result FileResult) { - if !fs.terminated { - fs.drain <- result - } -} diff --git a/src/magneticod/bittorrent/sinkMetadata.go b/src/magneticod/bittorrent/sinkMetadata.go deleted file mode 100644 index a5761d9..0000000 --- a/src/magneticod/bittorrent/sinkMetadata.go +++ /dev/null @@ -1,89 +0,0 @@ -package bittorrent - -import ( - "go.uber.org/zap" - "github.com/anacrolix/torrent" - - "magneticod/dht/mainline" - "persistence" -) - - -type Metadata struct { - InfoHash []byte - // Name should be thought of "Title" of the torrent. For single-file torrents, it is the name - // of the file, and for multi-file torrents, it is the name of the root directory. - Name string - TotalSize uint64 - DiscoveredOn int64 - // Files must be populated for both single-file and multi-file torrents! - Files []persistence.File - // Peers is the list of the "active" peers at the time of fetching metadata. Currently, it's - // always nil as anacrolix/torrent does not support returning list of peers for a given torrent, - // but in the future, this information can be useful for the CompletingCoordinator which can use - // those Peers to download the README file (if any found). - Peers []torrent.Peer - -} - - -type MetadataSink struct { - client *torrent.Client - drain chan Metadata - terminated bool - termination chan interface{} -} - - -func NewMetadataSink(laddr string) *MetadataSink { - ms := new(MetadataSink) - var err error - ms.client, err = torrent.NewClient(&torrent.Config{ - ListenAddr: laddr, - DisableTrackers: true, - DisablePEX: true, - // TODO: Should we disable DHT to force the client to use the peers we supplied only, or not? - NoDHT: true, - Seed: false, - - - }) - if err != nil { - zap.L().Fatal("Fetcher could NOT create a new torrent client!", zap.Error(err)) - } - ms.drain = make(chan Metadata) - ms.termination = make(chan interface{}) - return ms -} - - -func (ms *MetadataSink) Sink(res mainline.TrawlingResult) { - if ms.terminated { - zap.L().Panic("Trying to Sink() an already closed MetadataSink!") - } - - go ms.awaitMetadata(res.InfoHash, res.Peer) -} - - -func (ms *MetadataSink) Drain() <-chan Metadata { - if ms.terminated { - zap.L().Panic("Trying to Drain() an already closed MetadataSink!") - } - return ms.drain -} - - -func (ms *MetadataSink) Terminate() { - ms.terminated = true - close(ms.termination) - ms.client.Close() - close(ms.drain) -} - - -func (ms *MetadataSink) flush(result Metadata) { - if !ms.terminated { - ms.drain <- result - } -} diff --git a/src/magneticod/coordinators.go b/src/magneticod/coordinators.go deleted file mode 100644 index e4e59a9..0000000 --- a/src/magneticod/coordinators.go +++ /dev/null @@ -1,111 +0,0 @@ -package main - -import ( - "regexp" - "sync" - "time" - - "github.com/anacrolix/torrent" - - "persistence" - - "magneticod/bittorrent" - -) - -type completionRequest struct { - infoHash []byte - path string - peers []torrent.Peer - time time.Time -} - -type completionResult struct { - InfoHash []byte - Path string - Data []byte -} - -type CompletingCoordinator struct { - database persistence.Database - maxReadmeSize uint - sink *bittorrent.FileSink - queue chan completionRequest - queueMutex sync.Mutex - outputChan chan completionResult - readmeRegex *regexp.Regexp - terminated bool - termination chan interface{} -} - -type CompletingCoordinatorOpFlags struct { - LeechClAddr string - LeechMlAddr string - LeechTimeout time.Duration - ReadmeMaxSize uint - ReadmeRegex *regexp.Regexp -} - -func NewCompletingCoordinator(database persistence.Database, opFlags CompletingCoordinatorOpFlags) (cc *CompletingCoordinator) { - cc = new(CompletingCoordinator) - cc.database = database - cc.maxReadmeSize = opFlags.ReadmeMaxSize - cc.sink = bittorrent.NewFileSink(opFlags.LeechClAddr, opFlags.LeechMlAddr, opFlags.LeechTimeout) - cc.queue = make(chan completionRequest, 100) - cc.readmeRegex = opFlags.ReadmeRegex - cc.termination = make(chan interface{}) - return -} - -func (cc *CompletingCoordinator) Request(infoHash []byte, path string, peers []torrent.Peer) { - cc.queueMutex.Lock() - defer cc.queueMutex.Unlock() - - // If queue is full discard the oldest request as it is more likely to be outdated. - if len(cc.queue) == cap(cc.queue) { - <- cc.queue - } - - // Imagine, if this function [Request()] was called by another goroutine right when we were - // here: the moment where we removed the oldest entry in the queue to free a single space for - // the newest one. Imagine, now, that the second Request() call manages to add its own entry - // to the queue, making the current goroutine wait until the cc.queue channel is available. - // - // Hence to prevent that we use cc.queueMutex - - cc.queue <- completionRequest{ - infoHash: infoHash, - path: path, - peers: peers, - time: time.Now(), - } -} - -func (cc *CompletingCoordinator) Start() { - go cc.complete() -} - -func (cc *CompletingCoordinator) Output() <-chan completionResult { - return cc.outputChan -} - -func (cc *CompletingCoordinator) complete() { - for { - select { - case request := <-cc.queue: - // Discard requests older than 2 minutes. - // TODO: Instead of settling on 2 minutes as an arbitrary value, do some research to - // learn average peer lifetime in the BitTorrent network. - if time.Now().Sub(request.time) > 2 * time.Minute { - continue - } - cc.sink.Sink(request.infoHash, request.path, request.peers) - - case <-cc.termination: - break - - default: - cc.database.FindAnIncompleteTorrent(cc.readmeRegex, cc.maxReadmeSize) - } - } -} diff --git a/src/magneticod/dht/mainline/bloomFilter.go b/src/magneticod/dht/mainline/bloomFilter.go deleted file mode 100644 index 29fa90c..0000000 --- a/src/magneticod/dht/mainline/bloomFilter.go +++ /dev/null @@ -1,59 +0,0 @@ -package mainline - -import ( - "net" - "go.uber.org/zap" - "crypto/sha1" - "math/bits" - "math" -) - -const ( - k uint32 = 2 - m uint32 = 256 * 8 -) - -type BloomFilter struct { - filter [m/8]byte -} - -func (bf *BloomFilter) InsertIP(ip net.IP) { - if !(len(ip) == net.IPv4len || len(ip) == net.IPv6len) { - zap.S().Panicf("Attempted to insert an invalid IP to the bloom filter! %d", len(ip)) - } - - hash := sha1.Sum(ip) - - var index1, index2 uint32 - index1 = uint32(hash[0]) | uint32(hash[1]) << 8 - index2 = uint32(hash[2]) | uint32(hash[3]) << 8 - - // truncate index to m (11 bits required) - index1 %= m - index2 %= m - - // set bits at index1 and index2 - bf.filter[index1 / 8] |= 0x01 << (index1 % 8) - bf.filter[index2 / 8] |= 0x01 << (index2 % 8) -} - -func (bf *BloomFilter) Estimate() float64 { - // TODO: make it faster? - var nZeroes uint32 = 0 - for _, b := range bf.filter { - nZeroes += 8 - uint32(bits.OnesCount8(uint8(b))) - } - - var c uint32 - if m - 1 < nZeroes { - c = m - 1 - } else { - c = nZeroes - } - return math.Log(float64(c) / float64(m)) / (float64(k) * math.Log(1 - 1/float64(m))) -} - -func (bf *BloomFilter) Filter() (filterCopy [m/8]byte) { - copy(filterCopy[:], bf.filter[:]) - return filterCopy -} diff --git a/src/magneticod/dht/mainline/bloomFilter_test.go b/src/magneticod/dht/mainline/bloomFilter_test.go deleted file mode 100644 index 6df9fd6..0000000 --- a/src/magneticod/dht/mainline/bloomFilter_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package mainline - -import ( - "bytes" - "testing" - "encoding/hex" - "strings" - "fmt" -) - -func TestBEP33Filter(t *testing.T) { - bf := new(BloomFilter) - populateForBEP33(bf) - - resultingFilter := bf.Filter() - var expectedFilter [256]byte - hex.Decode(expectedFilter[:], []byte(strings.Replace( - "F6C3F5EA A07FFD91 BDE89F77 7F26FB2B FF37BDB8 FB2BBAA2 FD3DDDE7 BACFFF75 EE7CCBAE" + - "FE5EEDB1 FBFAFF67 F6ABFF5E 43DDBCA3 FD9B9FFD F4FFD3E9 DFF12D1B DF59DB53 DBE9FA5B" + - "7FF3B8FD FCDE1AFB 8BEDD7BE 2F3EE71E BBBFE93B CDEEFE14 8246C2BC 5DBFF7E7 EFDCF24F" + - "D8DC7ADF FD8FFFDF DDFFF7A4 BBEEDF5C B95CE81F C7FCFF1F F4FFFFDF E5F7FDCB B7FD79B3" + - "FA1FC77B FE07FFF9 05B7B7FF C7FEFEFF E0B8370B B0CD3F5B 7F2BD93F EB4386CF DD6F7FD5" + - "BFAF2E9E BFFFFEEC D67ADBF7 C67F17EF D5D75EBA 6FFEBA7F FF47A91E B1BFBB53 E8ABFB57" + - "62ABE8FF 237279BF EFBFEEF5 FFC5FEBF DFE5ADFF ADFEE1FB 737FFFFB FD9F6AEF FEEE76B6" + - "FD8F72EF", - " ", "", -1))) - if !bytes.Equal(resultingFilter[:], expectedFilter[:]) { - t.Fail() - } -} - -func TestBEP33Estimation(t *testing.T) { - bf := new(BloomFilter) - populateForBEP33(bf) - - // Because Go lacks a truncate function for floats... - if fmt.Sprintf("%.5f", bf.Estimate())[:9] != "1224.9308" { - t.Errorf("Expected 1224.9308 got %f instead!", bf.Estimate()) - } -} - -func populateForBEP33(bf *BloomFilter) { - // 192.0.2.0 to 192.0.2.255 (both ranges inclusive) - addr := []byte{192, 0, 2, 0} - for i := 0; i <= 255; i++ { - addr[3] = uint8(i) - bf.InsertIP(addr) - } - - // 2001:DB8:: to 2001:DB8::3E7 (both ranges inclusive) - addr = []byte{32, 1, 13, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} - for i := 0; i <= 2; i++ { - addr[14] = uint8(i) - for e := 0; e <= 255; e++ { - addr[15] = uint8(e) - bf.InsertIP(addr) - } - } - addr[14] = 3 - for e := 0; e <= 231; e++ { - addr[15] = uint8(e) - bf.InsertIP(addr) - } -} diff --git a/src/magneticod/main.go b/src/magneticod/main.go deleted file mode 100644 index 1d8885f..0000000 --- a/src/magneticod/main.go +++ /dev/null @@ -1,284 +0,0 @@ -package main - -import ( - "fmt" - "net" - "os" - "os/signal" - "regexp" - "time" - - "github.com/jessevdk/go-flags" - "github.com/pkg/profile" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - - "persistence" - - "magneticod/bittorrent" - "magneticod/dht" - "github.com/anacrolix/torrent/metainfo" -) - -type cmdFlags struct { - DatabaseURL string `long:"database" description:"URL of the database."` - - TrawlerMlAddrs []string `long:"trawler-ml-addr" description:"Address(es) to be used by trawling DHT (Mainline) nodes." default:"0.0.0.0:0"` - TrawlerMlInterval uint `long:"trawler-ml-interval" description:"Trawling interval in integer deciseconds (one tenth of a second)."` - - // TODO: is this even supported by anacrolix/torrent? - FetcherAddr string `long:"fetcher-addr" description:"Address(es) to be used by ephemeral peers fetching torrent metadata." default:"0.0.0.0:0"` - FetcherTimeout uint `long:"fetcher-timeout" description:"Number of integer seconds before a fetcher timeouts."` - - StatistMlAddrs []string `long:"statist-ml-addr" description:"Address(es) to be used by ephemeral nodes fetching latest statistics about individual torrents." default:"0.0.0.0:0"` - StatistMlTimeout uint `long:"statist-ml-timeout" description:"Number of integer seconds before a statist timeouts."` - - // TODO: is this even supported by anacrolix/torrent? - LeechClAddr string `long:"leech-cl-addr" description:"Address to be used by the peer fetching README files." default:"0.0.0.0:0"` - LeechMlAddr string `long:"leech-ml-addr" descrition:"Address to be used by the mainline DHT node for fetching README files." default:"0.0.0.0:0"` - LeechTimeout uint `long:"leech-timeout" description:"Number of integer seconds to pass before a leech timeouts." default:"300"` - ReadmeMaxSize uint `long:"readme-max-size" description:"Maximum size -which must be greater than zero- of a description file in bytes." default:"20480"` - ReadmeRegex string `long:"readme-regex" description:"Regular expression(s) which will be tested against the name of the README files, in the supplied order."` - - Verbose []bool `short:"v" long:"verbose" description:"Increases verbosity."` - - Profile string `long:"profile" description:"Enable profiling." default:""` - - // ==== OLD Flags ==== - - // DatabaseFile is akin to Database flag, except that it was used when SQLite was the only - // persistence backend ever conceived, so it's the path* to the database file, which was -by - // default- located in wherever appdata module on Python said: - // On GNU/Linux : `/home//.local/share/magneticod/database.sqlite3` - // On Windows : TODO? - // On MacOS (OS X) : TODO? - // On BSDs? : TODO? - // On anywhere else: TODO? - // TODO: Is the path* absolute or can be relative as well? -} - -const ( - PROFILE_BLOCK = 1 - PROFILE_CPU - PROFILE_MEM - PROFILE_MUTEX - PROFILE_A -) - -type opFlags struct { - DatabaseURL string - - TrawlerMlAddrs []string - TrawlerMlInterval time.Duration - - FetcherAddr string - FetcherTimeout time.Duration - - StatistMlAddrs []string - StatistMlTimeout time.Duration - - LeechClAddr string - LeechMlAddr string - LeechTimeout time.Duration - ReadmeMaxSize uint - ReadmeRegex *regexp.Regexp - - Verbosity int - - Profile string -} - -func main() { - loggerLevel := zap.NewAtomicLevel() - // Logging levels: ("debug", "info", "warn", "error", "dpanic", "panic", and "fatal"). - logger := zap.New(zapcore.NewCore( - zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), - zapcore.Lock(os.Stderr), - loggerLevel, - )) - defer logger.Sync() - zap.ReplaceGlobals(logger) - - defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop() - - zap.L().Info("magneticod v0.7.0 has been started.") - zap.L().Info("Copyright (C) 2017 Mert Bora ALPER .") - zap.L().Info("Dedicated to Cemile Binay, in whose hands I thrived.") - - // opFlags is the "operational flags" - opFlags := parseFlags() - - switch opFlags.Verbosity { - case 0: - loggerLevel.SetLevel(zap.WarnLevel) - case 1: - loggerLevel.SetLevel(zap.InfoLevel) - // Default: i.e. in case of 2 or more. - default: - loggerLevel.SetLevel(zap.DebugLevel) - } - - zap.ReplaceGlobals(logger) - - // Handle Ctrl-C gracefully. - interruptChan := make(chan os.Signal) - signal.Notify(interruptChan, os.Interrupt) - - database, err := persistence.MakeDatabase(opFlags.DatabaseURL) - if err != nil { - logger.Sugar().Fatalf("Could not open the database at `%s`: %s", opFlags.DatabaseURL, err.Error()) - } - - trawlingManager := dht.NewTrawlingManager(opFlags.TrawlerMlAddrs) - metadataSink := bittorrent.NewMetadataSink(opFlags.FetcherAddr) - completingCoordinator := NewCompletingCoordinator(database, CompletingCoordinatorOpFlags{ - LeechClAddr: opFlags.LeechClAddr, - LeechMlAddr: opFlags.LeechMlAddr, - LeechTimeout: opFlags.LeechTimeout, - ReadmeMaxSize: opFlags.ReadmeMaxSize, - ReadmeRegex: opFlags.ReadmeRegex, - }) - /* - refreshingCoordinator := NewRefreshingCoordinator(database, RefreshingCoordinatorOpFlags{ - - }) - */ - - for { - select { - case result := <-trawlingManager.Output(): - logger.Debug("result: ", zap.String("hash", result.InfoHash.String())) - exists, err := database.DoesTorrentExist(result.InfoHash[:]) - if err != nil { - zap.L().Fatal("Could not check whether torrent exists!", zap.Error(err)) - } else if !exists { - metadataSink.Sink(result) - } - - case metadata := <-metadataSink.Drain(): - if err := database.AddNewTorrent(metadata.InfoHash, metadata.Name, metadata.Files); err != nil { - logger.Sugar().Fatalf("Could not add new torrent %x to the database: %s", - metadata.InfoHash, err.Error()) - } - logger.Sugar().Infof("D I S C O V E R E D: `%s` %x", metadata.Name, metadata.InfoHash) - - if readmePath := findReadme(opFlags.ReadmeRegex, metadata.Files); readmePath != nil { - completingCoordinator.Request(metadata.InfoHash, *readmePath, metadata.Peers) - } - - case result := <-completingCoordinator.Output(): - database.AddReadme(result.InfoHash, result.Path, result.Data) - - case <-interruptChan: - trawlingManager.Terminate() - break - } - } -} - -func parseFlags() (opF opFlags) { - var cmdF cmdFlags - - _, err := flags.Parse(&cmdF) - if err != nil { - zap.S().Fatalf("Could not parse command-line flags! %s", err.Error()) - } - - // TODO: Check Database URL here - opF.DatabaseURL = cmdF.DatabaseURL - - if err = checkAddrs(cmdF.TrawlerMlAddrs); err != nil { - zap.S().Fatalf("Of argument (list) `trawler-ml-addr` %s", err.Error()) - } else { - opF.TrawlerMlAddrs = cmdF.TrawlerMlAddrs - } - - if cmdF.TrawlerMlInterval <= 0 { - zap.L().Fatal("Argument `trawler-ml-interval` must be greater than zero, if supplied.") - } else { - // 1 decisecond = 100 milliseconds = 0.1 seconds - opF.TrawlerMlInterval = time.Duration(cmdF.TrawlerMlInterval) * 100 * time.Millisecond - } - - if err = checkAddrs([]string{cmdF.FetcherAddr}); err != nil { - zap.S().Fatalf("Of argument `fetcher-addr` %s", err.Error()) - } else { - opF.FetcherAddr = cmdF.FetcherAddr - } - - if cmdF.FetcherTimeout <= 0 { - zap.L().Fatal("Argument `fetcher-timeout` must be greater than zero, if supplied.") - } else { - opF.FetcherTimeout = time.Duration(cmdF.FetcherTimeout) * time.Second - } - - if err = checkAddrs(cmdF.StatistMlAddrs); err != nil { - zap.S().Fatalf("Of argument (list) `statist-ml-addr` %s", err.Error()) - } else { - opF.StatistMlAddrs = cmdF.StatistMlAddrs - } - - if cmdF.StatistMlTimeout <= 0 { - zap.L().Fatal("Argument `statist-ml-timeout` must be greater than zero, if supplied.") - } else { - opF.StatistMlTimeout = time.Duration(cmdF.StatistMlTimeout) * time.Second - } - - if err = checkAddrs([]string{cmdF.LeechClAddr}); err != nil { - zap.S().Fatalf("Of argument `leech-cl-addr` %s", err.Error()) - } else { - opF.LeechClAddr = cmdF.LeechClAddr - } - - if err = checkAddrs([]string{cmdF.LeechMlAddr}); err != nil { - zap.S().Fatalf("Of argument `leech-ml-addr` %s", err.Error()) - } else { - opF.LeechMlAddr = cmdF.LeechMlAddr - } - - if cmdF.LeechTimeout <= 0 { - zap.L().Fatal("Argument `leech-timeout` must be greater than zero, if supplied.") - } else { - opF.LeechTimeout = time.Duration(cmdF.LeechTimeout) * time.Second - } - - if cmdF.ReadmeMaxSize <= 0 { - zap.L().Fatal("Argument `readme-max-size` must be greater than zero, if supplied.") - } else { - opF.ReadmeMaxSize = cmdF.ReadmeMaxSize - } - - opF.ReadmeRegex, err = regexp.Compile(cmdF.ReadmeRegex) - if err != nil { - zap.S().Fatalf("Argument `readme-regex` is not a valid regex: %s", err.Error()) - } - - opF.Verbosity = len(cmdF.Verbose) - opF.Profile = cmdF.Profile - - return -} - -func checkAddrs(addrs []string) error { - for i, addr := range addrs { - // We are using ResolveUDPAddr but it works equally well for checking TCPAddr(esses) as - // well. - _, err := net.ResolveUDPAddr("udp", addr) - if err != nil { - return fmt.Errorf("with %d(th) address `%s`: %s", i + 1, addr, err.Error()) - } - } - return nil -} - -// findReadme looks for a possible Readme file whose path is matched by the pathRegex. -// If there are multiple matches, the first one is returned. -// If there are no matches, nil returned. -func findReadme(pathRegex *regexp.Regexp, files []persistence.File) *string { - for _, file := range files { - if pathRegex.MatchString(file.Path) { - return &file.Path - } - } - return nil -} diff --git a/src/magneticod/persistence/database.go b/src/magneticod/persistence/database.go deleted file mode 100644 index ff866b1..0000000 --- a/src/magneticod/persistence/database.go +++ /dev/null @@ -1,385 +0,0 @@ -package persistence - -import ( - "bytes" - "database/sql" - "fmt" - "net/url" - "path" - "os" - - _ "github.com/go-sql-driver/mysql" - _ "github.com/mattn/go-sqlite3" - "go.uber.org/zap" - - "magneticod/bittorrent" - "regexp" -) - -type engineType uint8 - -const ( - SQLITE engineType = 0 - POSTGRESQL = 1 - MYSQL = 2 -) - -type Database struct { - database *sql.DB - engine engineType - newTorrents [] bittorrent.Metadata -} - -// NewDatabase creates a new Database. -// -// url either starts with "sqlite:" or "postgresql:" -func NewDatabase(rawurl string) (*Database, error) { - db := Database{} - - dbURL, err := url.Parse(rawurl) - if err != nil { - return nil, err - } - - switch dbURL.Scheme { - case "sqlite": - db.engine = SQLITE - dbDir, _ := path.Split(dbURL.Path) - if err := os.MkdirAll(dbDir, 0755); err != nil { - return nil, fmt.Errorf("for directory `%s`: %s", dbDir, err.Error()) - } - db.database, err = sql.Open("sqlite3", dbURL.Path) - - case "postgresql": - db.engine = POSTGRESQL - db.database, err = sql.Open("postgresql", rawurl) - - case "mysql": - db.engine = MYSQL - db.database, err = sql.Open("mysql", rawurl) - - default: - return nil, fmt.Errorf("unknown URI scheme (or malformed URI)!") - } - - // Check for errors from sql.Open() - if err != nil { - return nil, fmt.Errorf("error in sql.Open(): %s", err.Error()) - } - - if err = db.database.Ping(); err != nil { - return nil, fmt.Errorf("error in DB.Ping(): %s", err.Error()) - } - - if err := db.setupDatabase(); err != nil { - return nil, fmt.Errorf("error in setupDatabase(): %s", err.Error()) - } - - return &db, nil -} - - -func (db *Database) DoesTorrentExist(infoHash []byte) bool { - for _, torrent := range db.newTorrents { - if bytes.Equal(infoHash, torrent.InfoHash) { - return true; - } - } - - rows, err := db.database.Query("SELECT info_hash FROM torrents WHERE info_hash = ?;", infoHash) - if err != nil { - zap.L().Sugar().Fatalf("Could not query whether a torrent exists in the database! %s", err.Error()) - } - defer rows.Close() - - // If rows.Next() returns true, meaning that the torrent is in the database, return true; else - // return false. - return rows.Next() -} - -func (db *Database) FindAnIncompleteTorrent(pathRegex *regexp.Regexp, maxSize uint) error { - switch db.engine { - case SQLITE: - return db.findAnIncompleteTorrent_SQLite(pathRegex, maxSize) - - default: - zap.L().Fatal("Unknown database engine!", zap.Uint8("engine", uint8(db.engine))) - return nil - } -} - -func (db *Database) findAnIncompleteTorrent_SQLite(pathRegex *regexp.Regexp, maxSize uint) error { - // TODO: Prefer torrents with most seeders & leechs (i.e. most popular) - _, err := db.database.Query(` - SELECT torrents.info_hash, files.path FROM files WHERE files.path REGEXP ? - INNER JOIN torrents ON files.torrent_id = torrents.id LIMIT 1; - `) - if err != nil { - return err - } - - return nil -} - - -// AddNewTorrent adds a new torrent to the *queue* to be flushed to the persistent database. -func (db *Database) AddNewTorrent(torrent bittorrent.Metadata) error { - // Although we check whether the torrent exists in the database before asking MetadataSink to - // fetch its metadata, the torrent can also exists in the Sink before that. Now, if a torrent in - // the sink is still being fetched, that's still not a problem as we just add the new peer for - // the torrent and exit, but if the torrent is complete (i.e. its metadata) and if its waiting - // in the channel to be received, a race condition arises when we query the database and seeing - // that it doesn't exists there, add it to the sink. - // Hence check for the last time whether the torrent exists in the database, and only if not, - // add it. - if db.DoesTorrentExist(torrent.InfoHash) { - return nil; - } - - db.newTorrents = append(db.newTorrents, torrent) - - if len(db.newTorrents) >= 10 { - zap.L().Sugar().Debugf("newTorrents queue is full, attempting to commit %d torrents...", - len(db.newTorrents)) - if err := db.commitNewTorrents(); err != nil { - return err - } - } - - return nil -} - - -func (db *Database) AddReadme(infoHash []byte, path string, data []byte) error { - // TODO - return nil -} - - -func (db *Database) commitNewTorrents() error { - tx, err := db.database.Begin() - if err != nil { - return fmt.Errorf("sql.DB.Begin()! %s", err.Error()) - } - - var nTorrents, nFiles uint - nTorrents = uint(len(db.newTorrents)) - for i, torrent := range db.newTorrents { - zap.L().Sugar().Debugf("Flushing torrent %d of %d: `%s` (%x)...", - i + 1, len(db.newTorrents), torrent.Name, torrent.InfoHash) - res, err := tx.Exec("INSERT INTO torrents (info_hash, name, total_size, discovered_on) VALUES (?, ?, ?, ?);", - torrent.InfoHash, torrent.Name, torrent.TotalSize, torrent.DiscoveredOn) - if err != nil { - ourError := fmt.Errorf("error while INSERTing INTO torrent: %s", err.Error()) - if err := tx.Rollback(); err != nil { - return fmt.Errorf("%s\tmeanwhile, could not rollback the current transaction either! %s", ourError.Error(), err.Error()) - } - return ourError - } - var lastInsertId int64 - if lastInsertId, err = res.LastInsertId(); err != nil { - return fmt.Errorf("sql.Result.LastInsertId()! %s", err.Error()) - } - - for _, file := range torrent.Files { - zap.L().Sugar().Debugf("Flushing file `%s` (of torrent %x)", path.Join(file.Path...), torrent.InfoHash) - _, err := tx.Exec("INSERT INTO files (torrent_id, size, path) VALUES(?, ?, ?);", - lastInsertId, file.Length, path.Join(file.Path...)) - if err != nil { - ourError := fmt.Errorf("error while INSERTing INTO files: %s", err.Error()) - if err := tx.Rollback(); err != nil { - return fmt.Errorf("%s\tmeanwhile, could not rollback the current transaction either! %s", ourError.Error(), err.Error()) - } - return ourError - } - } - nFiles += uint(len(torrent.Files)) - } - - if err = tx.Commit(); err != nil { - return fmt.Errorf("sql.Tx.Commit()! %s", err.Error()) - } - - // Clear the queue - db.newTorrents = nil - - zap.L().Sugar().Infof("%d torrents (%d files) are flushed to the database successfully.", - nTorrents, nFiles) - return nil -} - -func (db *Database) Close() { - // Be careful to not to get into an infinite loop. =) - db.database.Close() -} - -func (db *Database) setupDatabase() error { - switch db.engine { - case SQLITE: - return setupSqliteDatabase(db.database) - - case POSTGRESQL: - zap.L().Fatal("setupDatabase() is not implemented for PostgreSQL yet!") - - case MYSQL: - return setupMySQLDatabase(db.database) - - default: - zap.L().Sugar().Fatalf("Unknown database engine value %d! (programmer error)", db.engine) - } - - return nil -} - -func setupSqliteDatabase(database *sql.DB) error { - // Enable Write-Ahead Logging for SQLite as "WAL provides more concurrency as readers do not - // block writers and a writer does not block readers. Reading and writing can proceed - // concurrently." - // Caveats: - // * Might be unsupported by OSes other than Windows and UNIXes. - // * Does not work over a network filesystem. - // * Transactions that involve changes against multiple ATTACHed databases are not atomic - // across all databases as a set. - // See: https://www.sqlite.org/wal.html - // - // Force SQLite to use disk, instead of memory, for all temporary files to reduce the memory - // footprint. - // - // Enable foreign key constraints in SQLite which are crucial to prevent programmer errors on - // our side. - _, err := database.Exec(` - PRAGMA journal_mode=WAL; - PRAGMA temp_store=1; - PRAGMA foreign_keys=ON; - `) - if err != nil { - return err - } - - tx, err := database.Begin() - if err != nil { - return err - } - - // Essential, and valid for all user_version`s: - // TODO: "torrent_id" column of the "files" table can be NULL, how can we fix this in a new schema? - _, err = tx.Exec(` - CREATE TABLE IF NOT EXISTS torrents ( - id INTEGER PRIMARY KEY, - info_hash BLOB NOT NULL UNIQUE, - name TEXT NOT NULL, - total_size INTEGER NOT NULL CHECK(total_size > 0), - discovered_on INTEGER NOT NULL CHECK(discovered_on > 0) - ); - CREATE TABLE IF NOT EXISTS files ( - id INTEGER PRIMARY KEY, - torrent_id INTEGER REFERENCES torrents ON DELETE CASCADE ON UPDATE RESTRICT, - size INTEGER NOT NULL, - path TEXT NOT NULL - ); - `) - if err != nil { - return err - } - - // Get the user_version: - res, err := tx.Query(`PRAGMA user_version;`) - if err != nil { - return err - } - var userVersion int; - res.Next() - res.Scan(&userVersion) - - switch userVersion { - // Upgrade from user_version 0 to 1 - // The Change: - // * `info_hash_index` is recreated as UNIQUE. - case 0: - zap.S().Warnf("Updating database schema from 0 to 1... (this might take a while)") - _, err = tx.Exec(` - DROP INDEX info_hash_index; - CREATE UNIQUE INDEX info_hash_index ON torrents (info_hash); - PRAGMA user_version = 1; - `) - if err != nil { - return err - } - fallthrough - // Upgrade from user_version 1 to 2 - // The Change: - // * Added `is_readme` and `content` columns to the `files` table, and the constraints & the - // the indices they entail. - // * Added unique index `readme_index` on `files` table. - case 1: - zap.S().Warnf("Updating database schema from 1 to 2... (this might take a while)") - // We introduce two new columns here: content BLOB, and is_readme INTEGER which we treat as - // a bool (hence the CHECK). - // The reason for the change is that as we introduce the new "readme" feature which - // downloads a readme file as a torrent descriptor, we needed to store it somewhere in the - // database with the following conditions: - // - // 1. There can be one and only one readme (content) for a given torrent; hence the - // UNIQUE INDEX on (torrent_id, is_description) (remember that SQLite treats each NULL - // value as distinct [UNIQUE], see https://sqlite.org/nulls.html). - // 2. We would like to keep the readme (content) associated with the file it came from; - // hence we modify the files table instead of the torrents table. - // - // Regarding the implementation details, following constraints arise: - // - // 1. The column is_readme is either NULL or 1, and if it is 1, then content column cannot - // be NULL (but might be an empty BLOB). Vice versa, if content column of a row is, - // NULL then is_readme must be NULL. - // - // This is to prevent unused content fields filling up the database, and to catch - // programmers' errors. - _, err = tx.Exec(` - ALTER TABLE files ADD COLUMN is_readme INTEGER CHECK (is_readme IS NULL OR is_readme=1); - ALTER TABLE files ADD COLUMN content BLOB CHECK((content IS NULL AND is_readme IS NULL) OR (content IS NOT NULL AND is_readme=1)); - CREATE UNIQUE INDEX readme_index ON files (torrent_id, is_readme); - PRAGMA user_version = 2; - `) - if err != nil { - return err - } - } - - if err = tx.Commit(); err != nil { - return err - } - - return nil -} - -func setupMySQLDatabase(database *sql.DB) error { - // Set strict mode to prevent silent truncation - _, err := database.Exec(`SET SESSION SQL_MODE = 'STRICT_ALL_TABLES';`) - if err != nil { - return err - } - - _, err = database.Exec( - `CREATE TABLE IF NOT EXISTS torrents (" - id INTEGER PRIMARY KEY AUTO_INCREMENT, - info_hash BINARY(20) NOT NULL UNIQUE, - name VARCHAR(1024) NOT NULL, - total_size BIGINT UNSIGNED NOT NULL, - discovered_on INTEGER UNSIGNED NOT NULL - ); - - ALTER TABLE torrents ADD INDEX info_hash_index (info_hash); - - CREATE TABLE IF NOT EXISTS files ( - id INTEGER PRIMARY KEY AUTO_INCREMENT, - torrent_id INTEGER REFERENCES torrents (id) ON DELETE CASCADE ON UPDATE RESTRICT, - size BIGINT NOT NULL, - path TEXT NOT NULL - );`, - ) - - if err != nil { - return err - } - - return nil -} diff --git a/src/magneticod/persistence/database_test.go b/src/magneticod/persistence/database_test.go deleted file mode 100644 index 98ffafe..0000000 --- a/src/magneticod/persistence/database_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package persistence - -import ( - "path" - "testing" -) - - -// TestPathJoin tests the assumption we made in flushNewTorrents() function where we assumed path -// separator to be the `/` (slash), and not `\` (backslash) character (which is used by Windows). -// -// Golang seems to use slash character on both platforms but we need to check that slash character -// is used in all cases. As a rule of thumb in secure programming, always check ONLY for the valid -// case AND IGNORE THE REST (e.g. do not check for backslashes but check for slashes). -func TestPathJoin(t *testing.T) { - if path.Join("a", "b", "c") != "a/b/c" { - t.Errorf("path.Join uses a different character than `/` (slash) character as path separator! (path: `%s`)", - path.Join("a", "b", "c")) - } -} diff --git a/src/magneticow/Gopkg.toml b/src/magneticow/Gopkg.toml deleted file mode 100644 index b769e79..0000000 --- a/src/magneticow/Gopkg.toml +++ /dev/null @@ -1,26 +0,0 @@ - -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - - -[[constraint]] - name = "github.com/gorilla/mux" - version = "1.4.0" diff --git a/src/magneticow/bindata.go b/src/magneticow/bindata.go deleted file mode 100644 index 24f10a3..0000000 --- a/src/magneticow/bindata.go +++ /dev/null @@ -1,262 +0,0 @@ -package main - -import ( - "fmt" - "io/ioutil" -) - -// bindata_read reads the given file from disk. It returns -// an error on failure. -func bindata_read(path, name string) ([]byte, error) { - buf, err := ioutil.ReadFile(path) - if err != nil { - err = fmt.Errorf("Error reading asset %s at %s: %v", name, path, err) - } - return buf, err -} - - -// templates_torrent_html reads file data from disk. -// It panics if something went wrong in the process. -func templates_torrent_html() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/templates/torrent.html", - "templates/torrent.html", - ) -} - -// templates_feed_xml reads file data from disk. -// It panics if something went wrong in the process. -func templates_feed_xml() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/templates/feed.xml", - "templates/feed.xml", - ) -} - -// templates_homepage_html reads file data from disk. -// It panics if something went wrong in the process. -func templates_homepage_html() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/templates/homepage.html", - "templates/homepage.html", - ) -} - -// templates_statistics_html reads file data from disk. -// It panics if something went wrong in the process. -func templates_statistics_html() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/templates/statistics.html", - "templates/statistics.html", - ) -} - -// templates_torrents_html reads file data from disk. -// It panics if something went wrong in the process. -func templates_torrents_html() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/templates/torrents.html", - "templates/torrents.html", - ) -} - -// static_scripts_plotly_v1_26_1_min_js reads file data from disk. -// It panics if something went wrong in the process. -func static_scripts_plotly_v1_26_1_min_js() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/scripts/plotly-v1.26.1.min.js", - "static/scripts/plotly-v1.26.1.min.js", - ) -} - -// static_scripts_statistics_js reads file data from disk. -// It panics if something went wrong in the process. -func static_scripts_statistics_js() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/scripts/statistics.js", - "static/scripts/statistics.js", - ) -} - -// static_scripts_torrent_js reads file data from disk. -// It panics if something went wrong in the process. -func static_scripts_torrent_js() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/scripts/torrent.js", - "static/scripts/torrent.js", - ) -} - -// static_styles_reset_css reads file data from disk. -// It panics if something went wrong in the process. -func static_styles_reset_css() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/styles/reset.css", - "static/styles/reset.css", - ) -} - -// static_styles_statistics_css reads file data from disk. -// It panics if something went wrong in the process. -func static_styles_statistics_css() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/styles/statistics.css", - "static/styles/statistics.css", - ) -} - -// static_styles_torrent_css reads file data from disk. -// It panics if something went wrong in the process. -func static_styles_torrent_css() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/styles/torrent.css", - "static/styles/torrent.css", - ) -} - -// static_styles_torrents_css reads file data from disk. -// It panics if something went wrong in the process. -func static_styles_torrents_css() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/styles/torrents.css", - "static/styles/torrents.css", - ) -} - -// static_styles_homepage_css reads file data from disk. -// It panics if something went wrong in the process. -func static_styles_homepage_css() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/styles/homepage.css", - "static/styles/homepage.css", - ) -} - -// static_styles_essential_css reads file data from disk. -// It panics if something went wrong in the process. -func static_styles_essential_css() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/styles/essential.css", - "static/styles/essential.css", - ) -} - -// static_assets_magnet_gif reads file data from disk. -// It panics if something went wrong in the process. -func static_assets_magnet_gif() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/assets/magnet.gif", - "static/assets/magnet.gif", - ) -} - -// static_assets_feed_png reads file data from disk. -// It panics if something went wrong in the process. -func static_assets_feed_png() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/assets/feed.png", - "static/assets/feed.png", - ) -} - -// static_fonts_notomono_license_ofl_txt reads file data from disk. -// It panics if something went wrong in the process. -func static_fonts_notomono_license_ofl_txt() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoMono/LICENSE_OFL.txt", - "static/fonts/NotoMono/LICENSE_OFL.txt", - ) -} - -// static_fonts_notomono_regular_ttf reads file data from disk. -// It panics if something went wrong in the process. -func static_fonts_notomono_regular_ttf() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoMono/Regular.ttf", - "static/fonts/NotoMono/Regular.ttf", - ) -} - -// static_fonts_notosansui_license_ofl_txt reads file data from disk. -// It panics if something went wrong in the process. -func static_fonts_notosansui_license_ofl_txt() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoSansUI/LICENSE_OFL.txt", - "static/fonts/NotoSansUI/LICENSE_OFL.txt", - ) -} - -// static_fonts_notosansui_bold_ttf reads file data from disk. -// It panics if something went wrong in the process. -func static_fonts_notosansui_bold_ttf() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoSansUI/Bold.ttf", - "static/fonts/NotoSansUI/Bold.ttf", - ) -} - -// static_fonts_notosansui_bolditalic_ttf reads file data from disk. -// It panics if something went wrong in the process. -func static_fonts_notosansui_bolditalic_ttf() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoSansUI/BoldItalic.ttf", - "static/fonts/NotoSansUI/BoldItalic.ttf", - ) -} - -// static_fonts_notosansui_italic_ttf reads file data from disk. -// It panics if something went wrong in the process. -func static_fonts_notosansui_italic_ttf() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoSansUI/Italic.ttf", - "static/fonts/NotoSansUI/Italic.ttf", - ) -} - -// static_fonts_notosansui_regular_ttf reads file data from disk. -// It panics if something went wrong in the process. -func static_fonts_notosansui_regular_ttf() ([]byte, error) { - return bindata_read( - "/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoSansUI/Regular.ttf", - "static/fonts/NotoSansUI/Regular.ttf", - ) -} - -// Asset loads and returns the asset for the given name. -// It returns an error if the asset could not be found or -// could not be loaded. -func Asset(name string) ([]byte, error) { - if f, ok := _bindata[name]; ok { - return f() - } - return nil, fmt.Errorf("Asset %s not found", name) -} - -// _bindata is a table, holding each asset generator, mapped to its name. -var _bindata = map[string] func() ([]byte, error) { - "templates/torrent.html": templates_torrent_html, - "templates/feed.xml": templates_feed_xml, - "templates/homepage.html": templates_homepage_html, - "templates/statistics.html": templates_statistics_html, - "templates/torrents.html": templates_torrents_html, - "static/scripts/plotly-v1.26.1.min.js": static_scripts_plotly_v1_26_1_min_js, - "static/scripts/statistics.js": static_scripts_statistics_js, - "static/scripts/torrent.js": static_scripts_torrent_js, - "static/styles/reset.css": static_styles_reset_css, - "static/styles/statistics.css": static_styles_statistics_css, - "static/styles/torrent.css": static_styles_torrent_css, - "static/styles/torrents.css": static_styles_torrents_css, - "static/styles/homepage.css": static_styles_homepage_css, - "static/styles/essential.css": static_styles_essential_css, - "static/assets/magnet.gif": static_assets_magnet_gif, - "static/assets/feed.png": static_assets_feed_png, - "static/fonts/NotoMono/LICENSE_OFL.txt": static_fonts_notomono_license_ofl_txt, - "static/fonts/NotoMono/Regular.ttf": static_fonts_notomono_regular_ttf, - "static/fonts/NotoSansUI/LICENSE_OFL.txt": static_fonts_notosansui_license_ofl_txt, - "static/fonts/NotoSansUI/Bold.ttf": static_fonts_notosansui_bold_ttf, - "static/fonts/NotoSansUI/BoldItalic.ttf": static_fonts_notosansui_bolditalic_ttf, - "static/fonts/NotoSansUI/Italic.ttf": static_fonts_notosansui_italic_ttf, - "static/fonts/NotoSansUI/Regular.ttf": static_fonts_notosansui_regular_ttf, - -} diff --git a/src/magneticow/data/static/styles/homepage.css b/src/magneticow/data/static/styles/homepage.css deleted file mode 100644 index dcca09c..0000000 --- a/src/magneticow/data/static/styles/homepage.css +++ /dev/null @@ -1,38 +0,0 @@ -main { - display: flex; - align-items: center; - align-content: center; - - height: calc(100vh - 2*16px - 0.833em - 23px); /* 100vh - body's padding(s) - footer margin - footer height */ - width: 100%; -} - -@media (max-width: 616px) { - main { - flex-direction: column; - justify-content: center; - align-items: flex-start; - } -} - -main div#magneticow { - white-space: nowrap; - margin: 0 0.5em 0.5em 0; -} - -main form { - max-width: 600px; - width: 100%; -} - -main form input { - width: 100%; -} - -main > div { - margin-right: 0.5em; -} - -footer { - margin-top: 0.833em; -} diff --git a/src/magneticow/data/templates/feed.xml b/src/magneticow/data/templates/feed.xml deleted file mode 100644 index 6a3c230..0000000 --- a/src/magneticow/data/templates/feed.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - {{ .Title }} - - {% for item in items %} - - {{ item.title }} - {{ item.DiscoveredOn }} - {{ item.info_hash }} - - - - {% endfor %} - - \ No newline at end of file diff --git a/src/magneticow/data/templates/torrents.html b/src/magneticow/data/templates/torrents.html deleted file mode 100644 index 4caa44d..0000000 --- a/src/magneticow/data/templates/torrents.html +++ /dev/null @@ -1,66 +0,0 @@ - - - - - {% if .search %}"{{.search}}"{% else %}Most recent torrents{% endif %} - magneticow - - - - - - -
-
magneticow​(pre-alpha)
-
- -
- -
-
- - - - - - - - - - - {% for torrent in torrents %} - - - - - - - {% endfor %} - -
NameSizeDiscovered on
- Magnet link{{ torrent.name }}{{ torrent.size }}{{ torrent.discovered_on }}
-
- - - \ No newline at end of file diff --git a/src/magneticow/main.go b/src/magneticow/main.go deleted file mode 100644 index 2bc09d8..0000000 --- a/src/magneticow/main.go +++ /dev/null @@ -1,108 +0,0 @@ -package main - -import ( - "html/template" - "log" - "net/http" - "strings" - - "github.com/gorilla/mux" - - "persistence" -) - -const N_TORRENTS = 20 - -var templates map[string]*template.Template -var database persistence.Database - -func main() { - router := mux.NewRouter() - router.HandleFunc("/", rootHandler) - router.HandleFunc("/torrents", torrentsHandler) - router.HandleFunc("/torrents/{infohash}", torrentsInfohashHandler) - router.HandleFunc("/torrents/{infohash}/{name}", torrentsInfohashNameHandler) - router.HandleFunc("/statistics", statisticsHandler) - router.PathPrefix("/static").HandlerFunc(staticHandler) - - router.HandleFunc("/feed", feedHandler) - - templates = make(map[string]*template.Template) - templates["feed"] = template.Must(template.New("feed").Parse(string(mustAsset("templates/feed.xml")))) - templates["homepage"] = template.Must(template.New("homepage").Parse(string(mustAsset("templates/homepage.html")))) - templates["statistics"] = template.Must(template.New("statistics").Parse(string(mustAsset("templates/statistics.html")))) - templates["torrent"] = template.Must(template.New("torrent").Parse(string(mustAsset("templates/torrent.html")))) - templates["torrents"] = template.Must(template.New("torrents").Parse(string(mustAsset("templates/torrents.html")))) - - var err error - database, err = persistence.MakeDatabase("sqlite3:///home/bora/.local/share/magneticod/database.sqlite3") - if err != nil { - panic(err.Error()) - } - - http.ListenAndServe(":8080", router) -} - - -func rootHandler(w http.ResponseWriter, r *http.Request) { - count, err := database.GetNumberOfTorrents() - if err != nil { - panic(err.Error()) - } - templates["homepage"].Execute(w, count) -} - - -func torrentsHandler(w http.ResponseWriter, r *http.Request) { - /* - newestTorrents, err := database.NewestTorrents(N_TORRENTS) - if err != nil { - panic(err.Error()) - } - templates["torrents"].Execute(w, nil) - */ -} - -func torrentsInfohashHandler(w http.ResponseWriter, r *http.Request) { - // redirect to torrents/{infohash}/name -} - - -func torrentsInfohashNameHandler(w http.ResponseWriter, r *http.Request) { - -} - - -func statisticsHandler(w http.ResponseWriter, r *http.Request) { - -} - - -func feedHandler(w http.ResponseWriter, r *http.Request) { - -} - -func staticHandler(w http.ResponseWriter, r *http.Request) { - data, err := Asset(r.URL.Path[1:]) - if err != nil { - http.NotFound(w, r) - return - } - - var contentType string - if strings.HasSuffix(r.URL.Path, ".css") { - contentType = "text/css; charset=utf-8" - } else { // fallback option - contentType = http.DetectContentType(data) - } - w.Header().Set("Content-Type", contentType) - w.Write(data) -} - -func mustAsset(name string) []byte { - data, err := Asset(name) - if err != nil { - log.Panicf("Could NOT access the requested resource `%s`: %s", name, err.Error()) - } - return data -} diff --git a/src/persistence/Gopkg.toml b/src/persistence/Gopkg.toml deleted file mode 100644 index 403c7cd..0000000 --- a/src/persistence/Gopkg.toml +++ /dev/null @@ -1,26 +0,0 @@ - -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - - -[[constraint]] - name = "go.uber.org/zap" - version = "1.6.0" diff --git a/src/persistence/interface.go b/src/persistence/interface.go deleted file mode 100644 index 5dc3eae..0000000 --- a/src/persistence/interface.go +++ /dev/null @@ -1,116 +0,0 @@ -package persistence - -import ( - "fmt" - "regexp" - "net/url" -) - -type Database interface { - Engine() databaseEngine - DoesTorrentExist(infoHash []byte) (bool, error) - // GiveAnIncompleteTorrentByInfoHash returns (*gives*) an incomplete -i.e. one that doesn't have - // readme downloaded yet- torrent from the database. - // GiveAnIncompleteTorrent might return a nil slice for infoHash, a nil string, and a nil err, - // meaning that no incomplete torrent could be found in the database (congrats!). - GiveAnIncompleteTorrent(pathRegex *regexp.Regexp, maxSize uint) (infoHash []byte, path string, err error) - GiveAStaleTorrent() (infoHash []byte, err error) - AddNewTorrent(infoHash []byte, name string, files []File) error - AddReadme(infoHash []byte, path string, content string) error - Close() error - - // GetNumberOfTorrents returns the number of torrents saved in the database. Might be an - // approximation. - GetNumberOfTorrents() (uint, error) - NewestTorrents(n uint) ([]TorrentMetadata, error) - SearchTorrents(query string, orderBy orderingCriteria, descending bool, mustHaveReadme bool) ([]TorrentMetadata, error) - // GetTorrents returns the TorrentExtMetadata for the torrent of the given infoHash. Might return - // nil, nil if the torrent does not exist in the database. - GetTorrent(infoHash []byte) (*TorrentMetadata, error) - GetFiles(infoHash []byte) ([]File, error) - GetReadme(infoHash []byte) (string, error) - GetStatistics(from ISO8601, period uint) (*Statistics, error) -} - -type orderingCriteria uint8 - -const ( - BY_NAME orderingCriteria = 1 - BY_SIZE = 2 - BY_DISCOVERED_ON = 3 - BY_N_FILES = 4 - BY_N_SEEDERS = 5 - BY_N_LEECHERS = 6 - BY_UPDATED_ON = 7 - BY_N_SEEDERS_TO_N_LEECHERS_RATIO = 8 - BY_N_SEEDERS_PLUS_N_LEECHERS = 9 -) - - -type statisticsGranularity uint8 -type ISO8601 string - -const ( - MINUTELY_STATISTICS statisticsGranularity = 1 - HOURLY_STATISTICS = 2 - DAILY_STATISTICS = 3 - WEEKLY_STATISTICS = 4 - MONTHLY_STATISTICS = 5 - YEARLY_STATISTICS = 6 -) - -type databaseEngine uint8 - -const ( - SQLITE3_ENGINE databaseEngine = 1 -) - -type Statistics struct { - Granularity statisticsGranularity - From ISO8601 - Period uint - - // All these slices below have the exact length equal to the Period. - NTorrentsDiscovered []uint - NFilesDiscovered []uint - NReadmesDownloaded []uint - NTorrentsUpdated []uint -} - -type File struct { - Size int64 - Path string -} - -type TorrentMetadata struct { - infoHash []byte - name string - size uint64 - discoveredOn int64 - hasReadme bool - nFiles uint - // values below 0 indicates that no data is available: - nSeeders int - nLeechers int - updatedOn int -} - -func MakeDatabase(rawURL string) (Database, error) { - url_, err := url.Parse(rawURL) - if err != nil { - return nil, err - } - - switch url_.Scheme { - case "sqlite3": - return makeSqlite3Database(url_) - - case "postgresql": - return nil, fmt.Errorf("postgresql is not yet supported!") - - case "mysql": - return nil, fmt.Errorf("mysql is not yet supported!") - } - - return nil, fmt.Errorf("unknown URI scheme (database engine)!") -} diff --git a/src/persistence/mysql.go b/src/persistence/mysql.go deleted file mode 100644 index dc7cf83..0000000 --- a/src/persistence/mysql.go +++ /dev/null @@ -1 +0,0 @@ -package persistence diff --git a/src/persistence/postgresql.go b/src/persistence/postgresql.go deleted file mode 100644 index dc7cf83..0000000 --- a/src/persistence/postgresql.go +++ /dev/null @@ -1 +0,0 @@ -package persistence