From 4c4dc1c1732585ab6ee49f7969328083e531a1f8 Mon Sep 17 00:00:00 2001 From: Bora Alper Date: Mon, 27 Aug 2018 11:15:01 +0300 Subject: [PATCH] added documentation, and Dockerfiles --- Dockerfile.magneticod | 28 +++++++++ Dockerfile.magneticow | 28 +++++++++ Makefile | 32 +++++++--- README.md | 36 ++++------- cmd/magneticod/README.md | 77 ++++++++++++++++++++++++ cmd/magneticow/README.md | 127 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 297 insertions(+), 31 deletions(-) create mode 100644 Dockerfile.magneticod create mode 100644 Dockerfile.magneticow create mode 100644 cmd/magneticod/README.md create mode 100644 cmd/magneticow/README.md diff --git a/Dockerfile.magneticod b/Dockerfile.magneticod new file mode 100644 index 0000000..6d60a65 --- /dev/null +++ b/Dockerfile.magneticod @@ -0,0 +1,28 @@ +# Source: https://blog.golang.org/docker + +# Start from a Debian image with the latest version of Go installed +# and a workspace (GOPATH) configured at /go. +FROM golang:1.10 + +# Copy the local package files to the container's workspace. +ADD ./Gopkg.toml /go/src/github.com/boramalper/magnetico/ +ADD ./Makefile /go/src/github.com/boramalper/magnetico/ +ADD ./pkg /go/src/github.com/boramalper/magnetico/pkg +ADD ./cmd/magneticod /go/src/github.com/boramalper/magnetico/cmd/magneticod + +# Build the outyet command inside the container. +# (You may fetch or manage dependencies here, +# either manually or with a tool like "godep".) +RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh +WORKDIR /go/src/github.com/boramalper/magnetico/ +RUN make ensure +RUN make test-persistence +RUN make test-magneticod +RUN make magneticod + +# Run the outyet command by default when the container starts. +ENTRYPOINT ["/go/bin/magneticod"] +CMD [] + +# Document that the service listens on port 8080. +EXPOSE 8080 diff --git a/Dockerfile.magneticow b/Dockerfile.magneticow new file mode 100644 index 0000000..dbea944 --- /dev/null +++ b/Dockerfile.magneticow @@ -0,0 +1,28 @@ +# Source: https://blog.golang.org/docker + +# Start from a Debian image with the latest version of Go installed +# and a workspace (GOPATH) configured at /go. +FROM golang:1.10 + +# Copy the local package files to the container's workspace. +ADD ./Gopkg.toml /go/src/github.com/boramalper/magnetico/ +ADD ./Makefile /go/src/github.com/boramalper/magnetico/ +ADD ./pkg /go/src/github.com/boramalper/magnetico/pkg +ADD ./cmd/magneticow /go/src/github.com/boramalper/magnetico/cmd/magneticow + +# Build the outyet command inside the container. +# (You may fetch or manage dependencies here, +# either manually or with a tool like "godep".) +RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh +WORKDIR /go/src/github.com/boramalper/magnetico/ +RUN make ensure +RUN make test-persistence +RUN make test-magneticow +RUN make magneticow + +# Run the outyet command by default when the container starts. +ENTRYPOINT ["/go/bin/magneticow"] +CMD [] + +# Document that the service listens on port 8080. +EXPOSE 8080 diff --git a/Makefile b/Makefile index 388f3c1..45389c1 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,38 @@ -.PHONY: test format +.PHONY: test format magneticod magneticow ensure test-magneticod test-magneticow test-persistence image image-magneticow image-magneticod -all: magneticod magneticow +all: test magneticod magneticow magneticod: - go install magnetico/magneticod + go install --tags fts5 "-ldflags=-s -w" github.com/boramalper/magnetico/cmd/magneticod magneticow: # TODO: minify files! - go-bindata -o="magneticow/bindata.go" -prefix="magneticow/data/" magneticow/data/... - go install magnetico/magneticow + go-bindata -o="cmd/magneticow/bindata.go" -prefix="cmd/magneticow/data/" cmd/magneticow/data/... + go install --tags fts5 "-ldflags=-s -w" github.com/boramalper/magnetico/cmd/magneticow -test: +image-magneticod: + docker build -t magneticod -f Dockerfile.magneticod . + +image-magneticow: + docker build -t magneticow -f Dockerfile.magneticow . + +image: image-magneticod image-magneticow + +# Download dependencies +ensure: + dep ensure -v + +test-magneticod: go test github.com/boramalper/magnetico/cmd/magneticod/... - @echo + +test-magneticow: go test github.com/boramalper/magnetico/cmd/magneticow/... - @echo + +test-persistence: go test github.com/boramalper/magnetico/pkg/persistence/... +test: test-persistence test-magneticod test-magneticow + format: gofmt -w cmd/magneticod gofmt -w cmd/magneticow diff --git a/README.md b/README.md index 6b5d62a..6d7e169 100644 --- a/README.md +++ b/README.md @@ -18,20 +18,20 @@ central entity*. ## Features - Easy installation & minimal requirements: - - Python 3.5+ and a few Python packages that is available on PyPI. - - Root access is *not* required to install. + - Pre-compiled static binaries and Docker images are provided. + - Root access is *not* required to install or to use. - Near-zero configuration: - - magneticod works out of the box, and magneticow requires minimal configuration to work with the - web server you choose. + - Both programs work out of the box, and **magneticow** can be used without a web-server too. - Detailed, step-by-step manual to guide you through the installation. - No reliance on any centralised entity: - - **magneticod** crawls the BitTorrent DHT by "going" from one node to another, and fetches the + - **magneticod** trawls the BitTorrent DHT by "going" from one node to another, and fetches the metadata using the nodes without using trackers. - Resilience: - Unlike client-server model that web applications use, P2P networks are *chaotic* and **magneticod** is designed to handle all the operational errors accordingly. -- High performance implementation: - - **magneticod** utilizes every bit of your bandwidth to discover as many infohashes & metadata as + - Currently on paper, wait for the v1.0! +- High performance implementation in Go: + - **magneticod** utilizes every bit of your resources to discover as many infohashes & metadata as possible. - Built-in lightweight web interface: - **magneticow** features a lightweight web interface to help you access the database without @@ -56,38 +56,28 @@ for torrents in the network, hence removing the need for centralised torrent web ## Installation Instructions > **WARNING:** > -> **magnetico** is still under active construction, and is considered *pre-alpha* software. Please +> **magnetico** is still under active construction, and is considered *alpha* software. Please > use **magnetico** suite with care and follow the installation instructions carefully to install > it & secure the installation. Feel perfectly free to send bug reports, suggestions, or whatever > comes to your mind to send to us through GitHub or personal e-mail. - -> **WARNING:** -> -> **magnetico** currently does NOT have any filtering system NOR it allows individual torrents to be -> removed from the database, and BitTorrent DHT network is full of the materials that are considered -> illegal in many countries (violence, pornography, copyright infringing content, and even -> child-pornography). If you are afraid of the legal consequences, or simply morally against -> (indirectly) assisting those content to spread around, follow the **magneticow** installation -> instructions carefully to password-protect the web-interface from others. - > **WARNING:** > > Do NOT clone the [repository](https://github.com/boramalper/magnetico) to install **magnetico**, > as it is never meant to be stable (except > [releases](https://github.com/boramalper/magnetico/releases) of course). -1. Install **magneticod** first by following its [installation instructions](magneticod/README.md). +1. Install **magneticod** first by following its [installation instructions](cmd/magneticod/README.md). 2. Install **magneticow** afterwards by following its - [installation instructions](magneticow/README.rst). + [installation instructions](cmd/magneticow/README.md). ## License -All the code is licensed under AGPLv3, unless otherwise stated in the source specific source. See -`COPYING` file for the full license text. +All the code is licensed under AGPLv3, unless stated otherwise specifically. See `COPYING` for +details. ---- Dedicated to Cemile Binay, in whose hands I thrived. -Bora M. ALPER <[bora@boramalper.org](mailto:bora@boramalper.org)> +Bora M. ALPER diff --git a/cmd/magneticod/README.md b/cmd/magneticod/README.md new file mode 100644 index 0000000..f4aaa98 --- /dev/null +++ b/cmd/magneticod/README.md @@ -0,0 +1,77 @@ +# magneticod +*Autonomous BitTorrent DHT crawler and metadata fetcher.* + +**magneticod** is the daemon that crawls the BitTorrent DHT network in the background to discover info hashes and +fetches metadata from the peers. + +## Installation + +### Requirements +- Decent Internet access (IPv4) + + **magneticod** uses UDP protocol to communicate with the nodes in the DHT network, and TCP to communicate with the + peers while fetching metadata. **Please make sure you have a healthy connection;** you can confirm this by checking at + the *connection status indicator* of your BitTorrent client: if it does not indicate any error (*e.g.* a misconfigured NAT), + **magneticod** should just work fine. + +### Installing the Pre-Compiled Static Binary +You can find the latest pre-compiled static binaries on [GitHub](https://github.com/boramalper/magnetico/releases) +for versions from v0.7.0 onwards. + +### Installing the Docker Image +Docker images are provided on [Docker Hub](https://hub.docker.com/r/boramalper/magnetico/tags/) at +the repository `boramalper/magnetico`. Images are tagged as `magneticod-vMAJOR.MINOR.PATCH`. + +## Setup +1. (Optional, **requires root**) Disable iptables for a specified port: + + ```bash + iptables -I OUTPUT -t raw -p udp --sport PORT_NUMBER -j NOTRACK + iptables -I PREROUTING -t raw -p udp --dport PORT_NUMBER -j NOTRACK + ``` + + This is to prevent excessive number of ``EPERM`` "Operation not permitted" errors, which also has a negative impact + on the performance. + +### Configuration +Configuration file can be found at: + +- **On Linux** + + ~/.config/magneticod/configuration.toml + + +## Usage +### Database +**magneticod** is designed to be able to use different database engines to store its data, but +currently only SQLite 3 is supported. The database file can be found in: + +- **On Linux** + + ~/.local/share/magneticod/ + +**magneticod** uses write-ahead logging (WAL) for its database, so there might be multiple +files while it is operating, but ``database.sqlite3`` is *the database*. + +### Using the Docker Image +You need to mount + +- the data directory (`~/.local/share/magneticod/` on Linux, see the previous sections) +- the configuration file at (`~/.config/magneticod/configuration.toml` on Linux, see the previous sections) + +hence run: + + ```bash + docker run \ + -v ~/.local/share/magneticod:/root/.local/share/magneticod/ \ + -v ~/.config/magneticod/configuration.toml:/root/.config/magneticod/configuration.toml \ + magneticod + ``` + +__Tip:__ Containers that you terminate won't be removed; run +`docker rm $(docker ps -q -f status=exited)` to remove exited containers. + +### Remark About the Network Usage +**magneticod** does *not* have any built-in rate limiter *yet*, and it will literally suck the hell out of your +bandwidth. Unless you are running **magneticod** on a separate machine dedicated for it, you might want to consider +starting it manually only when network load is low (e.g. when you are at work or sleeping at night). diff --git a/cmd/magneticow/README.md b/cmd/magneticow/README.md new file mode 100644 index 0000000..e490b5a --- /dev/null +++ b/cmd/magneticow/README.md @@ -0,0 +1,127 @@ +# magneticow +*Lightweight web interface for **magneticod**.* + +**magneticow** is a lightweight web interface to search and to browse the torrents that its counterpart (**magneticod**) +discovered. + +## Installation + +### Installing the Pre-Compiled Static Binary +You can find the latest pre-compiled static binaries on [GitHub](https://github.com/boramalper/magnetico/releases) +for versions from v0.7.0 onwards. + +### Installing the Docker Image +Docker images are provided on [Docker Hub](https://hub.docker.com/r/boramalper/magnetico/tags/) at +the repository `boramalper/magnetico`. Images are tagged as `magneticow-vMAJOR.MINOR.PATCH`. + +## Setup +### Configuration +Configuration file can be found at: + +- **On Linux** + + ~/.config/magneticow/configuration.toml + +### Setting Password-Protection +If you'd like to password-protect the access to **magneticow**, you need to store the credentials +in file + +- **On Linux:** + + ~/.config/magneticow/credentials + +- **On macOS (OS X):** + + TODO PATH HERE + +`credentials` file must consist of lines of the following format + + : + +where + +- `` must start with a small-case (`[a-z]`) ASCII character, might contain non-consecutive + underscores except at the end, and consists of small-case a-z characters and digits 0-9. +- `` is the output of the well-known bcrypt function. + +You can use `htpasswd` (part of `apache2-utils` on Ubuntu) to create lines: + +``` +$ htpasswd -bnBC 12 "USERNAME" "PASSWORD" +USERNAME:$2y$12$YE01LZ8jrbQbx6c0s2hdZO71dSjn2p/O9XsYJpz.5968yCysUgiaG +``` + +### Warnings +1. **magnetico** currently does NOT have any filtering system NOR it allows individual torrents to be removed from the + database, and BitTorrent DHT network is full of the materials that are considered illegal in many countries + (violence, pornography, copyright infringing content, and even child-pornography). If you are afraid of the legal + consequences, or simply morally against (indirectly) assisting those content to spread around, follow the + **magneticow** installation instructions carefully to password-protect the web-interface from others. + +2. **magneticow** uses HTTP Basic Authentication, meaning that your username and password will be + transmitted in plain-text for every request. Configuring **magneticow** to serve behind a + web-server with HTTPS enabled is strongly recommended, but unfortunately not described here. You + can use [Let's Encrypt](https://letsencrypt.org/) to get a certificate for free. + +3. **magneticow** is *NOT* designed to scale, and will fail miserably if you try to use it like a public torrent + website. This is a *deliberate* technical decision, not a bug or something to be fixed; *another* web interface with + more features to support such use cases and scalability *might* be developed, but **magneticow** will NEVER be the + case. + +## Usage +### Using the Docker Image +You need to mount + +- the data directory (`~/.local/share/magneticod/` on Linux, see [magneticod's Manual](../magneticod/README.md)) +- the configuration file at (`~/.config/magneticow/configuration.toml` on Linux, see the previous sections) + +hence run: + + ```bash + docker run \ + -v ~/.local/share/magneticod:/root/.local/share/magneticod/ \ + -v ~/.config/magneticow/configuration.toml:/root/.config/magneticow/configuration.toml \ + magneticow + ``` + +### Searching +* Only the **titles** of the torrents are being searched. +* Search is case-insensitive. +* Titles that includes terms that are separated by space are returned from the search: + + Example: ``king bad`` returns ``Stephen King - The Bazaar of Bad Dreams`` + + * If you would like terms to appear in the exact order you wrote them, enclose them in double quotes: + + Example: ``"king bad"`` returns ``George Benson - Good King Bad`` +* Use asteriks (``*``) to denote prefixes: + + Example: ``The Godf*`` returns ``Francis Ford Coppola - The Godfather`` + + Asteriks works inside the double quotes too! +* Use caret (``^``) to indicate that the term it prefixes must be the first word in the title: + + Example: ``linux`` returns ``Arch Linux`` while ``^linux`` would return ``Linux Mint`` + + * Caret works **inside** the double quotes **but not outside**: + + Right: ``"^ubuntu linux"`` + + Wrong: ``^"ubuntu linux"`` +* You can use ``AND``, ``OR`` and ``NOT`` and also parentheses for more complex queries: + + Example: ``programming NOT (windows OR "os x" OR macos)`` + + Beware that the terms are all-caps and MUST be so. + +| Operator | Syntax Precedence | +|----------|-----------------------------------------| +| `NOT` | Highest precedence (tightest grouping). | +| `AND` | | +| `OR` | Lowest precedence (loosest grouping). | + +### REST-ful HTTP API + +**magneticow** offers a REST-ful HTTP API that is capable of everything the web interface can do. + +See the [API documentation on Swaggerhub](https://app.swaggerhub.com/apis/boramalper/magneticow-api/v0.1).