rnhmjoj/magnetico
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

another halfway-through commit, ignore...

Browse Source
This commit is contained in:
Bora M. Alper 2017-10-04 14:07:48 +01:00
parent 620043f48c
commit 828e4691da
49 changed files with 1571 additions and 366 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -2,6 +2,8 @@ src/magneticod/vendor
src/magneticod/Gopkg.lock src/magneticod/Gopkg.lock
src/magneticow/vendor src/magneticow/vendor
src/magneticow/Gopkg.lock src/magneticow/Gopkg.lock
src/persistence/vendor
src/persistence/Gopkg.lock
# Created by https://www.gitignore.io/api/linux,python,pycharm # Created by https://www.gitignore.io/api/linux,python,pycharm

9
.travis.yml
View File

@ -1,15 +1,16 @@
sudo: enabled sudo: enabled
dist: xenial dist: xenial
language: python language: go
python: go:
- "3.5" - 1.8.3
# versions 3.6.0 and 3.6.1 have bugs that affect magneticod
before_install: before_install:
- "sudo apt-get update -qq" - "sudo apt-get update -qq"
- "sudo apt-get install python3-dev" - "sudo apt-get install python3-dev"
- "pip3 install mypy pylint" - "pip3 install mypy pylint"
- ""
install: install:
- "pip3 install ./magneticod" - "pip3 install ./magneticod"
- "pip3 install ./magneticow" - "pip3 install ./magneticow"

27
magneticow/magneticow/static/styles/homepage.css
View File

@ -1,27 +0,0 @@
main {
display: flex;
align-items: center;
align-content: center;
height: calc(100vh - 2 * 3em);
width: calc(100vw - 2 * 3em);
}
main form {
max-width: 600px;
width: 100%;
margin-left: 0.5em;
}
main form input {
width: 100%;
}
main > div {
margin-right: 0.5em;
}
footer {
margin-top: 0.833em;
}

23
magneticow/magneticow/templates/homepage.html
View File

@ -1,23 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>magneticow</title>
<link rel="stylesheet" href=" {{ url_for('static', filename='styles/reset.css') }} ">
<link rel="stylesheet" href=" {{ url_for('static', filename='styles/essential.css') }} ">
<link rel="stylesheet" href=" {{ url_for('static', filename='styles/homepage.css') }} ">
<!-- <script src="script.js"></script> -->
</head>
<body>
<main>
<div><b>magnetico<sup>w</sup></b>&#8203;<sub>(pre-alpha)</sub></div>
<form action="/torrents" method="get" autocomplete="off" role="search">
<input type="search" name="search" placeholder="Search the BitTorrent DHT" autofocus>
</form>
</main>
<footer>
~{{ "{:,}".format(n_torrents) }} torrents available (see the <a href="/statistics">statistics</a>).
</footer>
</body>
</html>

90
magneticow/magneticow/templates/torrents.html
View File

@ -1,90 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{% if search %}"{{search}}"{% else %}Most recent torrents{% endif %} - magneticow</title>
<link rel="stylesheet" href=" {{ url_for('static', filename='styles/reset.css') }} ">
<link rel="stylesheet" href=" {{ url_for('static', filename='styles/essential.css') }} ">
<link rel="stylesheet" href=" {{ url_for('static', filename='styles/torrents.css') }} ">
<!-- <script src="script.js"></script> -->
</head>
<body>
<header>
<div><a href="/"><b>magnetico<sup>w</sup></b></a>&#8203;<sub>(pre-alpha)</sub></div>
<form action="/torrents" method="get" autocomplete="off" role="search">
<input type="search" name="search" placeholder="Search the BitTorrent DHT" value="{{ search }}">
</form>
<div>
<a href="{{ subscription_url }}"><img src="{{ url_for('static', filename='assets/feed.png') }}"
alt="feed icon" title="subscribe" /> subscribe</a>
</div>
</header>
<main>
<table>
<thead>
<tr>
<th><!-- Magnet link --></th>
<th>
{% if sorted_by == "name ASC" %}
<a href="/torrents/?search={{ search }}&sort_by=name+DESC">Name ▲</a>
{% elif sorted_by == "name DESC" %}
<a href="/torrents/?search={{ search }}&sort_by=name+ASC">Name ▼</a>
{% else %}
<a href="/torrents/?search={{ search }}&sort_by=name+ASC">Name</a>
{% endif %}
</th>
<th>
{% if sorted_by == "total_size ASC" %}
<a href="/torrents/?search={{ search }}&sort_by=total_size+DESC">Size ▲</a>
{% elif sorted_by == "total_size DESC" %}
<a href="/torrents/?search={{ search }}&sort_by=total_size+ASC">Size ▼</a>
{% else %}
<a href="/torrents/?search={{ search }}&sort_by=total_size+ASC">Size</a>
{% endif %}
</th>
<th>
{% if sorted_by == "discovered_on ASC" %}
<a href="/torrents/?search={{ search }}&sort_by=discovered_on+DESC">Discovered on ▲</a>
{% elif sorted_by == "discovered_on DESC" %}
<a href="/torrents/?search={{ search }}&sort_by=discovered_on+ASC">Discovered on ▼</a>
{% else %}
<a href="/torrents/?search={{ search }}&sort_by=discovered_on+DESC">Discovered on</a>
{% endif %}
</th>
</tr>
</thead>
<tbody>
{% for torrent in torrents %}
<tr>
<td><a href="magnet:?xt=urn:btih:{{ torrent.info_hash }}&dn={{ torrent.name }}">
<img src="{{ url_for('static', filename='assets/magnet.gif') }}" alt="Magnet link"
title="Download this torrent using magnet" /></a></td>
<td><a href="/torrents/{{ torrent.info_hash }}/{{ torrent.name }}">{{ torrent.name }}</a></td>
<td>{{ torrent.size }}</td>
<td>{{ torrent.discovered_on }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</main>
<footer>
<form action="/torrents" method="get">
<button {% if page == 0 %}disabled{% endif %}>Previous</button>
<input type="text" name="search" value="{{ search }}" hidden>
{% if sorted_by %}
<input type="text" name="sort_by" value="{{ sorted_by }}" hidden>
{% endif %}
<input type="number" name="page" value="{{ page - 1 }}" hidden>
</form>
<form action="/torrents" method="get">
<button {% if not next_page_exists %}disabled{% endif %}>Next</button>
<input type="text" name="search" value="{{ search }}" hidden>
{% if sorted_by %}
<input type="text" name="sort_by" value="{{ sorted_by }}" hidden>
{% endif %}
<input type="number" name="page" value="{{ page + 1 }}" hidden>
</form>
</footer>
</body>
</html>

75
src/magneticod/bittorrent/operations.go
View File

@ -8,6 +8,9 @@ import (
"github.com/anacrolix/torrent" "github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo" "github.com/anacrolix/torrent/metainfo"
"go.uber.org/zap" "go.uber.org/zap"
"os"
"path"
"persistence"
) )
@ -40,48 +43,46 @@ func (ms *MetadataSink) awaitMetadata(infoHash metainfo.Hash, peer torrent.Peer)
return return
} }
var files []metainfo.FileInfo var files []persistence.File
if len(info.Files) == 0 { for _, file := range info.Files {
if strings.ContainsRune(info.Name, '/') { files = append(files, persistence.File{
// A single file torrent cannot have any '/' characters in its name. We treat it as Size: file.Length,
// illegal. Path: file.DisplayPath(info),
zap.L().Sugar().Debugf("!!!! illegal character in name! \"%s\"", info.Name) })
return
}
files = []metainfo.FileInfo{{Length: info.Length, Path:[]string{info.Name}}}
} else {
// TODO: We have to make sure that anacrolix/torrent checks for '/' character in file paths
// before concatenating them. This is currently assumed here. We should write a test for it.
files = info.Files
} }
var totalSize uint64 var totalSize uint64
for _, file := range files { for _, file := range files {
if file.Length < 0 { if file.Size < 0 {
// All files' sizes must be greater than or equal to zero, otherwise treat them as // All files' sizes must be greater than or equal to zero, otherwise treat them as
// illegal and ignore. // illegal and ignore.
zap.L().Sugar().Debugf("!!!! file size zero or less! \"%s\" (%d)", file.Path, file.Length) zap.L().Sugar().Debugf("!!!! file size zero or less! \"%s\" (%d)", file.Path, file.Size)
return return
} }
totalSize += uint64(file.Length) totalSize += uint64(file.Size)
} }
ms.flush(Metadata{ ms.flush(Metadata{
InfoHash: infoHash[:], InfoHash: infoHash[:],
Name: info.Name, Name: info.Name,
TotalSize: totalSize, TotalSize: totalSize,
DiscoveredOn: time.Now().Unix(), DiscoveredOn: time.Now().Unix(),
Files: files, Files: files,
Peers: nil,
}) })
} }
func (fs *FileSink) awaitFile(infoHash []byte, filePath string, peer *torrent.Peer) { func (fs *FileSink) awaitFile(req *FileRequest) {
// Remove the download directory of the torrent after the operation is completed.
// TODO: what if RemoveAll() returns error, do we care, and if we do, how to handle it?
defer os.RemoveAll(path.Join(fs.baseDownloadDir, string(req.InfoHash)))
var infoHash_ [20]byte var infoHash_ [20]byte
copy(infoHash_[:], infoHash) copy(infoHash_[:], req.InfoHash)
t, isNew := fs.client.AddTorrentInfoHash(infoHash_) t, isNew := fs.client.AddTorrentInfoHash(infoHash_)
if peer != nil { if len(req.Peers) > 0 {
t.AddPeers([]torrent.Peer{*peer}) t.AddPeers(req.Peers)
} }
if !isNew { if !isNew {
// Return immediately if we are trying to await on an ongoing file-downloading operation. // Return immediately if we are trying to await on an ongoing file-downloading operation.
@ -90,7 +91,7 @@ func (fs *FileSink) awaitFile(infoHash []byte, filePath string, peer *torrent.Pe
} }
// Setup & start the timeout timer. // Setup & start the timeout timer.
timeout := time.After(fs.timeout) timeout := time.After(fs.timeoutDuration)
// Once we return from this function, drop the torrent from the client. // Once we return from this function, drop the torrent from the client.
// TODO: Check if dropping a torrent also cancels any outstanding read operations? // TODO: Check if dropping a torrent also cancels any outstanding read operations?
@ -105,7 +106,7 @@ func (fs *FileSink) awaitFile(infoHash []byte, filePath string, peer *torrent.Pe
var match *torrent.File var match *torrent.File
for _, file := range t.Files() { for _, file := range t.Files() {
if file.Path() == filePath { if file.Path() == req.Path {
match = &file match = &file
} else { } else {
file.Cancel() file.Cancel()
@ -117,13 +118,12 @@ func (fs *FileSink) awaitFile(infoHash []byte, filePath string, peer *torrent.Pe
zap.L().Warn( zap.L().Warn(
"The leech (FileSink) has been requested to download a file which does not exist!", "The leech (FileSink) has been requested to download a file which does not exist!",
zap.ByteString("torrent", infoHash), zap.ByteString("torrent", req.InfoHash),
zap.String("requestedFile", filePath), zap.String("requestedFile", req.Path),
zap.Strings("allFiles", filePaths), zap.Strings("allFiles", filePaths),
) )
} }
reader := t.NewReader() reader := t.NewReader()
defer reader.Close() defer reader.Close()
@ -133,18 +133,17 @@ func (fs *FileSink) awaitFile(infoHash []byte, filePath string, peer *torrent.Pe
select { select {
case fileData := <-fileDataChan: case fileData := <-fileDataChan:
if fileData != nil { if fileData != nil {
fs.flush(File{ fs.flush(FileResult{
torrentInfoHash: infoHash, Request: req,
path: match.Path(), FileData: fileData,
data: fileData,
}) })
} }
case <- timeout: case <- timeout:
zap.L().Debug( zap.L().Debug(
"Timeout while downloading a file!", "Timeout while downloading a file!",
zap.ByteString("torrent", infoHash), zap.ByteString("torrent", req.InfoHash),
zap.String("file", filePath), zap.String("file", req.Path),
) )
} }
} }
@ -156,9 +155,10 @@ func downloadFile(file torrent.File, reader *torrent.Reader, fileDataChan chan<-
fileData := make([]byte, file.Length()) fileData := make([]byte, file.Length())
n, err := readSeeker.Read(fileData) n, err := readSeeker.Read(fileData)
if int64(n) != file.Length() { if int64(n) != file.Length() {
infoHash := file.Torrent().InfoHash()
zap.L().Debug( zap.L().Debug(
"Not all of a file could be read!", "Not all of a file could be read!",
zap.ByteString("torrent", file.Torrent().InfoHash()[:]), zap.ByteString("torrent", infoHash[:]),
zap.String("file", file.Path()), zap.String("file", file.Path()),
zap.Int64("fileLength", file.Length()), zap.Int64("fileLength", file.Length()),
zap.Int("n", n), zap.Int("n", n),
@ -167,10 +167,11 @@ func downloadFile(file torrent.File, reader *torrent.Reader, fileDataChan chan<-
return return
} }
if err != nil { if err != nil {
infoHash := file.Torrent().InfoHash()
zap.L().Debug( zap.L().Debug(
"Error while downloading a file!", "Error while downloading a file!",
zap.Error(err), zap.Error(err),
zap.ByteString("torrent", file.Torrent().InfoHash()[:]), zap.ByteString("torrent", infoHash[:]),
zap.String("file", file.Path()), zap.String("file", file.Path()),
zap.Int64("fileLength", file.Length()), zap.Int64("fileLength", file.Length()),
zap.Int("n", n), zap.Int("n", n),

54
src/magneticod/bittorrent/sinkFile.go
View File

@ -12,28 +12,33 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
) )
type FileRequest struct {
type File struct{ InfoHash []byte
torrentInfoHash []byte Path string
path string Peers []torrent.Peer
data []byte
} }
type FileResult struct {
// Request field is the original Request
Request *FileRequest
FileData []byte
}
type FileSink struct { type FileSink struct {
baseDownloadDir string
client *torrent.Client client *torrent.Client
drain chan File drain chan FileResult
terminated bool terminated bool
termination chan interface{} termination chan interface{}
timeout time.Duration timeoutDuration time.Duration
} }
// NewFileSink creates a new FileSink. // NewFileSink creates a new FileSink.
// //
// cAddr : client address // cAddr : client address
// mlAddr: mainline DHT node address // mlAddr: mainline DHT node address
func NewFileSink(cAddr, mlAddr string, timeout time.Duration) *FileSink { func NewFileSink(cAddr, mlAddr string, timeoutDuration time.Duration) *FileSink {
fs := new(FileSink) fs := new(FileSink)
mlUDPAddr, err := net.ResolveUDPAddr("udp", mlAddr) mlUDPAddr, err := net.ResolveUDPAddr("udp", mlAddr)
@ -49,6 +54,11 @@ func NewFileSink(cAddr, mlAddr string, timeout time.Duration) *FileSink {
return nil return nil
} }
fs.baseDownloadDir = path.Join(
appdirs.UserCacheDir("magneticod", "", "", true),
"downloads",
)
fs.client, err = torrent.NewClient(&torrent.Config{ fs.client, err = torrent.NewClient(&torrent.Config{
ListenAddr: cAddr, ListenAddr: cAddr,
DisableTrackers: true, DisableTrackers: true,
@ -57,10 +67,7 @@ func NewFileSink(cAddr, mlAddr string, timeout time.Duration) *FileSink {
Passive: true, Passive: true,
NoSecurity: true, NoSecurity: true,
}, },
DefaultStorage: storage.NewFileByInfoHash(path.Join( DefaultStorage: storage.NewFileByInfoHash(fs.baseDownloadDir),
appdirs.UserCacheDir("magneticod", "", "", true),
"downloads",
)),
}) })
if err != nil { if err != nil {
zap.L().Fatal("Leech could NOT create a new torrent client!", zap.Error(err)) zap.L().Fatal("Leech could NOT create a new torrent client!", zap.Error(err))
@ -68,21 +75,26 @@ func NewFileSink(cAddr, mlAddr string, timeout time.Duration) *FileSink {
return nil return nil
} }
fs.drain = make(chan File) fs.drain = make(chan FileResult)
fs.termination = make(chan interface{}) fs.termination = make(chan interface{})
fs.timeout = timeout fs.timeoutDuration = timeoutDuration
return fs return fs
} }
// peer field is optional and might be nil.
// peer might be nil func (fs *FileSink) Sink(infoHash []byte, path string, peers []torrent.Peer) {
func (fs *FileSink) Sink(infoHash []byte, filePath string, peer *torrent.Peer) { if fs.terminated {
go fs.awaitFile(infoHash, filePath, peer) zap.L().Panic("Trying to Sink() an already closed FileSink!")
}
go fs.awaitFile(&FileRequest{
InfoHash: infoHash,
Path: path,
Peers: peers,
})
} }
func (fs *FileSink) Drain() <-chan FileResult {
func (fs *FileSink) Drain() <-chan File {
if fs.terminated { if fs.terminated {
zap.L().Panic("Trying to Drain() an already closed FileSink!") zap.L().Panic("Trying to Drain() an already closed FileSink!")
} }
@ -98,7 +110,7 @@ func (fs *FileSink) Terminate() {
} }
func (fs *FileSink) flush(result File) { func (fs *FileSink) flush(result FileResult) {
if !fs.terminated { if !fs.terminated {
fs.drain <- result fs.drain <- result
} }

16
src/magneticod/bittorrent/sinkMetadata.go
View File

@ -1,13 +1,11 @@
package bittorrent package bittorrent
import ( import (
"net"
"go.uber.org/zap" "go.uber.org/zap"
"github.com/anacrolix/torrent" "github.com/anacrolix/torrent"
"github.com/anacrolix/torrent/metainfo"
"magneticod/dht/mainline" "magneticod/dht/mainline"
"persistence"
) )
@ -19,7 +17,13 @@ type Metadata struct {
TotalSize uint64 TotalSize uint64
DiscoveredOn int64 DiscoveredOn int64
// Files must be populated for both single-file and multi-file torrents! // Files must be populated for both single-file and multi-file torrents!
Files []metainfo.FileInfo Files []persistence.File
// Peers is the list of the "active" peers at the time of fetching metadata. Currently, it's
// always nil as anacrolix/torrent does not support returning list of peers for a given torrent,
// but in the future, this information can be useful for the CompletingCoordinator which can use
// those Peers to download the README file (if any found).
Peers []torrent.Peer
} }
@ -31,11 +35,11 @@ type MetadataSink struct {
} }
func NewMetadataSink(laddr net.TCPAddr) *MetadataSink { func NewMetadataSink(laddr string) *MetadataSink {
ms := new(MetadataSink) ms := new(MetadataSink)
var err error var err error
ms.client, err = torrent.NewClient(&torrent.Config{ ms.client, err = torrent.NewClient(&torrent.Config{
ListenAddr: laddr.String(), ListenAddr: laddr,
DisableTrackers: true, DisableTrackers: true,
DisablePEX: true, DisablePEX: true,
// TODO: Should we disable DHT to force the client to use the peers we supplied only, or not? // TODO: Should we disable DHT to force the client to use the peers we supplied only, or not?

111
src/magneticod/coordinators.go Normal file
View File

@ -0,0 +1,111 @@
package main
import (
"regexp"
"sync"
"time"
"github.com/anacrolix/torrent"
"persistence"
"magneticod/bittorrent"
)
type completionRequest struct {
infoHash []byte
path string
peers []torrent.Peer
time time.Time
}
type completionResult struct {
InfoHash []byte
Path string
Data []byte
}
type CompletingCoordinator struct {
database persistence.Database
maxReadmeSize uint
sink *bittorrent.FileSink
queue chan completionRequest
queueMutex sync.Mutex
outputChan chan completionResult
readmeRegex *regexp.Regexp
terminated bool
termination chan interface{}
}
type CompletingCoordinatorOpFlags struct {
LeechClAddr string
LeechMlAddr string
LeechTimeout time.Duration
ReadmeMaxSize uint
ReadmeRegex *regexp.Regexp
}
func NewCompletingCoordinator(database persistence.Database, opFlags CompletingCoordinatorOpFlags) (cc *CompletingCoordinator) {
cc = new(CompletingCoordinator)
cc.database = database
cc.maxReadmeSize = opFlags.ReadmeMaxSize
cc.sink = bittorrent.NewFileSink(opFlags.LeechClAddr, opFlags.LeechMlAddr, opFlags.LeechTimeout)
cc.queue = make(chan completionRequest, 100)
cc.readmeRegex = opFlags.ReadmeRegex
cc.termination = make(chan interface{})
return
}
func (cc *CompletingCoordinator) Request(infoHash []byte, path string, peers []torrent.Peer) {
cc.queueMutex.Lock()
defer cc.queueMutex.Unlock()
// If queue is full discard the oldest request as it is more likely to be outdated.
if len(cc.queue) == cap(cc.queue) {
<- cc.queue
}
// Imagine, if this function [Request()] was called by another goroutine right when we were
// here: the moment where we removed the oldest entry in the queue to free a single space for
// the newest one. Imagine, now, that the second Request() call manages to add its own entry
// to the queue, making the current goroutine wait until the cc.queue channel is available.
//
// Hence to prevent that we use cc.queueMutex
cc.queue <- completionRequest{
infoHash: infoHash,
path: path,
peers: peers,
time: time.Now(),
}
}
func (cc *CompletingCoordinator) Start() {
go cc.complete()
}
func (cc *CompletingCoordinator) Output() <-chan completionResult {
return cc.outputChan
}
func (cc *CompletingCoordinator) complete() {
for {
select {
case request := <-cc.queue:
// Discard requests older than 2 minutes.
// TODO: Instead of settling on 2 minutes as an arbitrary value, do some research to
// learn average peer lifetime in the BitTorrent network.
if time.Now().Sub(request.time) > 2 * time.Minute {
continue
}
cc.sink.Sink(request.infoHash, request.path, request.peers)
case <-cc.termination:
break
default:
cc.database.FindAnIncompleteTorrent(cc.readmeRegex, cc.maxReadmeSize)
}
}
}

59
src/magneticod/dht/mainline/bloomFilter.go Normal file
View File

@ -0,0 +1,59 @@
package mainline
import (
"net"
"go.uber.org/zap"
"crypto/sha1"
"math/bits"
"math"
)
const (
k uint32 = 2
m uint32 = 256 * 8
)
type BloomFilter struct {
filter [m/8]byte
}
func (bf *BloomFilter) InsertIP(ip net.IP) {
if !(len(ip) == net.IPv4len || len(ip) == net.IPv6len) {
zap.S().Panicf("Attempted to insert an invalid IP to the bloom filter! %d", len(ip))
}
hash := sha1.Sum(ip)
var index1, index2 uint32
index1 = uint32(hash[0]) | uint32(hash[1]) << 8
index2 = uint32(hash[2]) | uint32(hash[3]) << 8
// truncate index to m (11 bits required)
index1 %= m
index2 %= m
// set bits at index1 and index2
bf.filter[index1 / 8] |= 0x01 << (index1 % 8)
bf.filter[index2 / 8] |= 0x01 << (index2 % 8)
}
func (bf *BloomFilter) Estimate() float64 {
// TODO: make it faster?
var nZeroes uint32 = 0
for _, b := range bf.filter {
nZeroes += 8 - uint32(bits.OnesCount8(uint8(b)))
}
var c uint32
if m - 1 < nZeroes {
c = m - 1
} else {
c = nZeroes
}
return math.Log(float64(c) / float64(m)) / (float64(k) * math.Log(1 - 1/float64(m)))
}
func (bf *BloomFilter) Filter() (filterCopy [m/8]byte) {
copy(filterCopy[:], bf.filter[:])
return filterCopy
}

64
src/magneticod/dht/mainline/bloomFilter_test.go Normal file
View File

@ -0,0 +1,64 @@
package mainline
import (
"bytes"
"testing"
"encoding/hex"
"strings"
"fmt"
)
func TestBEP33Filter(t *testing.T) {
bf := new(BloomFilter)
populateForBEP33(bf)
resultingFilter := bf.Filter()
var expectedFilter [256]byte
hex.Decode(expectedFilter[:], []byte(strings.Replace(
"F6C3F5EA A07FFD91 BDE89F77 7F26FB2B FF37BDB8 FB2BBAA2 FD3DDDE7 BACFFF75 EE7CCBAE" +
"FE5EEDB1 FBFAFF67 F6ABFF5E 43DDBCA3 FD9B9FFD F4FFD3E9 DFF12D1B DF59DB53 DBE9FA5B" +
"7FF3B8FD FCDE1AFB 8BEDD7BE 2F3EE71E BBBFE93B CDEEFE14 8246C2BC 5DBFF7E7 EFDCF24F" +
"D8DC7ADF FD8FFFDF DDFFF7A4 BBEEDF5C B95CE81F C7FCFF1F F4FFFFDF E5F7FDCB B7FD79B3" +
"FA1FC77B FE07FFF9 05B7B7FF C7FEFEFF E0B8370B B0CD3F5B 7F2BD93F EB4386CF DD6F7FD5" +
"BFAF2E9E BFFFFEEC D67ADBF7 C67F17EF D5D75EBA 6FFEBA7F FF47A91E B1BFBB53 E8ABFB57" +
"62ABE8FF 237279BF EFBFEEF5 FFC5FEBF DFE5ADFF ADFEE1FB 737FFFFB FD9F6AEF FEEE76B6" +
"FD8F72EF",
" ", "", -1)))
if !bytes.Equal(resultingFilter[:], expectedFilter[:]) {
t.Fail()
}
}
func TestBEP33Estimation(t *testing.T) {
bf := new(BloomFilter)
populateForBEP33(bf)
// Because Go lacks a truncate function for floats...
if fmt.Sprintf("%.5f", bf.Estimate())[:9] != "1224.9308" {
t.Errorf("Expected 1224.9308 got %f instead!", bf.Estimate())
}
}
func populateForBEP33(bf *BloomFilter) {
// 192.0.2.0 to 192.0.2.255 (both ranges inclusive)
addr := []byte{192, 0, 2, 0}
for i := 0; i <= 255; i++ {
addr[3] = uint8(i)
bf.InsertIP(addr)
}
// 2001:DB8:: to 2001:DB8::3E7 (both ranges inclusive)
addr = []byte{32, 1, 13, 184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
for i := 0; i <= 2; i++ {
addr[14] = uint8(i)
for e := 0; e <= 255; e++ {
addr[15] = uint8(e)
bf.InsertIP(addr)
}
}
addr[14] = 3
for e := 0; e <= 231; e++ {
addr[15] = uint8(e)
bf.InsertIP(addr)
}
}

27
src/magneticod/dht/mainline/codec.go
View File

@ -14,6 +14,7 @@ import (
"github.com/anacrolix/torrent/bencode" "github.com/anacrolix/torrent/bencode"
"github.com/anacrolix/missinggo/iter" "github.com/anacrolix/missinggo/iter"
"regexp" "regexp"
"github.com/willf/bloom"
) )
@ -46,6 +47,21 @@ type QueryArguments struct {
Port int `bencode:"port,omitempty"` Port int `bencode:"port,omitempty"`
// Use senders apparent DHT port // Use senders apparent DHT port
ImpliedPort int `bencode:"implied_port,omitempty"` ImpliedPort int `bencode:"implied_port,omitempty"`
// Indicates whether the querying node is seeding the torrent it announces.
// Defined in BEP 33 "DHT Scrapes" for `announce_peer` queries.
Seed int `bencode:"seed,omitempty"`
// If 1, then the responding node should try to fill the `values` list with non-seed items on a
// best-effort basis."
// Defined in BEP 33 "DHT Scrapes" for `get_peers` queries.
NoSeed int `bencode:"noseed,omitempty"`
// If 1, then the responding node should add two fields to the "r" dictionary in the response:
// - `BFsd`: Bloom Filter (256 bytes) representing all stored seeds for that infohash
// - `BFpe`: Bloom Filter (256 bytes) representing all stored peers (leeches) for that
// infohash
// Defined in BEP 33 "DHT Scrapes" for `get_peers` queries.
Scrape int `bencode:"noseed,omitempty"`
} }
@ -58,6 +74,15 @@ type ResponseValues struct {
Token []byte `bencode:"token,omitempty"` Token []byte `bencode:"token,omitempty"`
// Torrent peers // Torrent peers
Values []CompactPeer `bencode:"values,omitempty"` Values []CompactPeer `bencode:"values,omitempty"`
// If `scrape` is set to 1 in the `get_peers` query then the responding node should add the
// below two fields to the "r" dictionary in the response:
// Defined in BEP 33 "DHT Scrapes" for responses to `get_peers` queries.
// Bloom Filter (256 bytes) representing all stored seeds for that infohash:
BFsd *bloom.BloomFilter `bencode:"BFsd,omitempty"`
// Bloom Filter (256 bytes) representing all stored peers (leeches) for that infohash:
BFpe *bloom.BloomFilter `bencode:"BFpe,omitempty"`
// TODO: write marshallers for those fields above ^^
} }
@ -204,7 +229,7 @@ func (cni *CompactNodeInfo) UnmarshalBinary(b []byte) error {
func (cnis CompactNodeInfos) MarshalBencode() ([]byte, error) { func (cnis CompactNodeInfos) MarshalBencode() ([]byte, error) {
ret := make([]byte, 0) // TODO: this doesn't look idiomatic at all, is this the right way? var ret []byte
for _, cni := range cnis { for _, cni := range cnis {
ret = append(ret, cni.MarshalBinary()...) ret = append(ret, cni.MarshalBinary()...)

2
src/magneticod/dht/mainline/protocol.go
View File

@ -31,7 +31,7 @@ type ProtocolEventHandlers struct {
} }
func NewProtocol(laddr net.UDPAddr, eventHandlers ProtocolEventHandlers) (p *Protocol) { func NewProtocol(laddr string, eventHandlers ProtocolEventHandlers) (p *Protocol) {
p = new(Protocol) p = new(Protocol)
p.transport = NewTransport(laddr, p.onMessage) p.transport = NewTransport(laddr, p.onMessage)
p.eventHandlers = eventHandlers p.eventHandlers = eventHandlers

2
src/magneticod/dht/mainline/service.go
View File

@ -38,7 +38,7 @@ type TrawlingServiceEventHandlers struct {
} }
func NewTrawlingService(laddr net.UDPAddr, eventHandlers TrawlingServiceEventHandlers) *TrawlingService { func NewTrawlingService(laddr string, eventHandlers TrawlingServiceEventHandlers) *TrawlingService {
service := new(TrawlingService) service := new(TrawlingService)
service.protocol = NewProtocol( service.protocol = NewProtocol(
laddr, laddr,

13
src/magneticod/dht/mainline/transport.go
View File

@ -10,8 +10,8 @@ import (
type Transport struct { type Transport struct {
conn *net.UDPConn conn *net.UDPConn
laddr net.UDPAddr laddr *net.UDPAddr
started bool started bool
// OnMessage is the function that will be called when Transport receives a packet that is // OnMessage is the function that will be called when Transport receives a packet that is
@ -21,10 +21,13 @@ type Transport struct {
} }
func NewTransport(laddr net.UDPAddr, onMessage func(*Message, net.Addr)) (*Transport) { func NewTransport(laddr string, onMessage func(*Message, net.Addr)) (*Transport) {
transport := new(Transport) transport := new(Transport)
transport.onMessage = onMessage transport.onMessage = onMessage
transport.laddr = laddr var err error; transport.laddr, err = net.ResolveUDPAddr("udp", laddr)
if err != nil {
zap.L().Panic("Could not resolve the UDP address for the trawler!", zap.Error(err))
}
return transport return transport
} }
@ -45,7 +48,7 @@ func (t *Transport) Start() {
t.started = true t.started = true
var err error var err error
t.conn, err = net.ListenUDP("udp", &t.laddr) t.conn, err = net.ListenUDP("udp", t.laddr)
if err != nil { if err != nil {
zap.L().Fatal("Could NOT create a UDP socket!", zap.Error(err)) zap.L().Fatal("Could NOT create a UDP socket!", zap.Error(err))
} }

37
src/magneticod/dht/managers.go
View File

@ -1,10 +1,6 @@
package dht package dht
import ( import "magneticod/dht/mainline"
"magneticod/dht/mainline"
"net"
"github.com/bradfitz/iter"
)
type TrawlingManager struct { type TrawlingManager struct {
@ -14,29 +10,20 @@ type TrawlingManager struct {
} }
func NewTrawlingManager(mlAddrs []net.UDPAddr) *TrawlingManager { func NewTrawlingManager(mlAddrs []string) *TrawlingManager {
manager := new(TrawlingManager) manager := new(TrawlingManager)
manager.output = make(chan mainline.TrawlingResult) manager.output = make(chan mainline.TrawlingResult)
if mlAddrs != nil { if mlAddrs == nil {
for _, addr := range mlAddrs { mlAddrs = []string{"0.0.0.0:0"}
manager.services = append(manager.services, mainline.NewTrawlingService( }
addr, for _, addr := range mlAddrs {
mainline.TrawlingServiceEventHandlers{ manager.services = append(manager.services, mainline.NewTrawlingService(
OnResult: manager.onResult, addr,
}, mainline.TrawlingServiceEventHandlers{
)) OnResult: manager.onResult,
} },
} else { ))
addr := net.UDPAddr{IP: []byte("\x00\x00\x00\x00"), Port: 0}
for range iter.N(1) {
manager.services = append(manager.services, mainline.NewTrawlingService(
addr,
mainline.TrawlingServiceEventHandlers{
OnResult: manager.onResult,
},
))
}
} }
for _, service := range manager.services { for _, service := range manager.services {

156
src/magneticod/main.go
View File

@ -1,20 +1,23 @@
package main package main
import ( import (
"fmt"
"net" "net"
"os" "os"
"os/signal" "os/signal"
"regexp" "regexp"
"time"
"github.com/jessevdk/go-flags" "github.com/jessevdk/go-flags"
"github.com/pkg/profile" "github.com/pkg/profile"
"go.uber.org/zap" "go.uber.org/zap"
"go.uber.org/zap/zapcore" "go.uber.org/zap/zapcore"
"persistence"
"magneticod/bittorrent" "magneticod/bittorrent"
"magneticod/dht" "magneticod/dht"
"fmt" "github.com/anacrolix/torrent/metainfo"
"time"
) )
type cmdFlags struct { type cmdFlags struct {
@ -35,14 +38,13 @@ type cmdFlags struct {
LeechMlAddr string `long:"leech-ml-addr" descrition:"Address to be used by the mainline DHT node for fetching README files." default:"0.0.0.0:0"` LeechMlAddr string `long:"leech-ml-addr" descrition:"Address to be used by the mainline DHT node for fetching README files." default:"0.0.0.0:0"`
LeechTimeout uint `long:"leech-timeout" description:"Number of integer seconds to pass before a leech timeouts." default:"300"` LeechTimeout uint `long:"leech-timeout" description:"Number of integer seconds to pass before a leech timeouts." default:"300"`
ReadmeMaxSize uint `long:"readme-max-size" description:"Maximum size -which must be greater than zero- of a description file in bytes." default:"20480"` ReadmeMaxSize uint `long:"readme-max-size" description:"Maximum size -which must be greater than zero- of a description file in bytes." default:"20480"`
ReadmeRegexes []string `long:"readme-regex" description:"Regular expression(s) which will be tested against the name of the README files, in the supplied order."` ReadmeRegex string `long:"readme-regex" description:"Regular expression(s) which will be tested against the name of the README files, in the supplied order."`
Verbose []bool `short:"v" long:"verbose" description:"Increases verbosity."` Verbose []bool `short:"v" long:"verbose" description:"Increases verbosity."`
Profile string `long:"profile" description:"Enable profiling." default:""` Profile string `long:"profile" description:"Enable profiling." default:""`
// ==== Deprecated Flags ==== // ==== OLD Flags ====
// TODO: don't even support deprecated flags!
// DatabaseFile is akin to Database flag, except that it was used when SQLite was the only // DatabaseFile is akin to Database flag, except that it was used when SQLite was the only
// persistence backend ever conceived, so it's the path* to the database file, which was -by // persistence backend ever conceived, so it's the path* to the database file, which was -by
@ -53,7 +55,6 @@ type cmdFlags struct {
// On BSDs? : TODO? // On BSDs? : TODO?
// On anywhere else: TODO? // On anywhere else: TODO?
// TODO: Is the path* absolute or can be relative as well? // TODO: Is the path* absolute or can be relative as well?
// DatabaseFile string
} }
const ( const (
@ -70,19 +71,17 @@ type opFlags struct {
TrawlerMlAddrs []string TrawlerMlAddrs []string
TrawlerMlInterval time.Duration TrawlerMlInterval time.Duration
// TODO: is this even supported by anacrolix/torrent?
FetcherAddr string FetcherAddr string
FetcherTimeout time.Duration FetcherTimeout time.Duration
StatistMlAddrs []string StatistMlAddrs []string
StatistMlTimeout time.Duration StatistMlTimeout time.Duration
// TODO: is this even supported by anacrolix/torrent?
LeechClAddr string LeechClAddr string
LeechMlAddr string LeechMlAddr string
LeechTimeout time.Duration LeechTimeout time.Duration
ReadmeMaxSize uint ReadmeMaxSize uint
ReadmeRegexes []*regexp.Regexp ReadmeRegex *regexp.Regexp
Verbosity int Verbosity int
@ -90,12 +89,12 @@ type opFlags struct {
} }
func main() { func main() {
atom := zap.NewAtomicLevel() loggerLevel := zap.NewAtomicLevel()
// Logging levels: ("debug", "info", "warn", "error", "dpanic", "panic", and "fatal"). // Logging levels: ("debug", "info", "warn", "error", "dpanic", "panic", and "fatal").
logger := zap.New(zapcore.NewCore( logger := zap.New(zapcore.NewCore(
zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()),
zapcore.Lock(os.Stderr), zapcore.Lock(os.Stderr),
atom, loggerLevel,
)) ))
defer logger.Sync() defer logger.Sync()
zap.ReplaceGlobals(logger) zap.ReplaceGlobals(logger)
@ -111,81 +110,70 @@ func main() {
switch opFlags.Verbosity { switch opFlags.Verbosity {
case 0: case 0:
atom.SetLevel(zap.WarnLevel) loggerLevel.SetLevel(zap.WarnLevel)
case 1: case 1:
atom.SetLevel(zap.InfoLevel) loggerLevel.SetLevel(zap.InfoLevel)
// Default: i.e. in case of 2 or more. // Default: i.e. in case of 2 or more.
default: default:
atom.SetLevel(zap.DebugLevel) loggerLevel.SetLevel(zap.DebugLevel)
} }
zap.ReplaceGlobals(logger) zap.ReplaceGlobals(logger)
/*
updating_manager := nil
statistics_sink := nil
completing_manager := nil
file_sink := nil
*/
// Handle Ctrl-C gracefully. // Handle Ctrl-C gracefully.
interrupt_chan := make(chan os.Signal) interruptChan := make(chan os.Signal)
signal.Notify(interrupt_chan, os.Interrupt) signal.Notify(interruptChan, os.Interrupt)
database, err := NewDatabase(opFlags.Database) database, err := persistence.MakeDatabase(opFlags.DatabaseURL)
if err != nil { if err != nil {
logger.Sugar().Fatalf("Could not open the database at `%s`: %s", opFlags.Database, err.Error()) logger.Sugar().Fatalf("Could not open the database at `%s`: %s", opFlags.DatabaseURL, err.Error())
} }
trawlingManager := dht.NewTrawlingManager(opFlags.MlTrawlerAddrs) trawlingManager := dht.NewTrawlingManager(opFlags.TrawlerMlAddrs)
metadataSink := bittorrent.NewMetadataSink(opFlags.FetcherAddr) metadataSink := bittorrent.NewMetadataSink(opFlags.FetcherAddr)
fileSink := bittorrent.NewFileSink() completingCoordinator := NewCompletingCoordinator(database, CompletingCoordinatorOpFlags{
LeechClAddr: opFlags.LeechClAddr,
go func() { LeechMlAddr: opFlags.LeechMlAddr,
for { LeechTimeout: opFlags.LeechTimeout,
select { ReadmeMaxSize: opFlags.ReadmeMaxSize,
case result := <-trawlingManager.Output(): ReadmeRegex: opFlags.ReadmeRegex,
logger.Debug("result: ", zap.String("hash", result.InfoHash.String())) })
if !database.DoesExist(result.InfoHash[:]) {
metadataSink.Sink(result)
}
case metadata := <-metadataSink.Drain():
logger.Sugar().Infof("D I S C O V E R E D: `%s` %x",
metadata.Name, metadata.InfoHash)
if err := database.AddNewTorrent(metadata); err != nil {
logger.Sugar().Fatalf("Could not add new torrent %x to the database: %s",
metadata.InfoHash, err.Error())
}
case <-interrupt_chan:
trawlingManager.Terminate()
break
}
}
}()
go func() {
}()
go func() {
}()
/* /*
for { refreshingCoordinator := NewRefreshingCoordinator(database, RefreshingCoordinatorOpFlags{
select {
case updating_manager.Output(): })
case statistics_sink.Sink():
case completing_manager.Output():
case file_sink.Sink():
*/ */
<-interrupt_chan for {
select {
case result := <-trawlingManager.Output():
logger.Debug("result: ", zap.String("hash", result.InfoHash.String()))
exists, err := database.DoesTorrentExist(result.InfoHash[:])
if err != nil {
zap.L().Fatal("Could not check whether torrent exists!", zap.Error(err))
} else if !exists {
metadataSink.Sink(result)
}
case metadata := <-metadataSink.Drain():
if err := database.AddNewTorrent(metadata.InfoHash, metadata.Name, metadata.Files); err != nil {
logger.Sugar().Fatalf("Could not add new torrent %x to the database: %s",
metadata.InfoHash, err.Error())
}
logger.Sugar().Infof("D I S C O V E R E D: `%s` %x", metadata.Name, metadata.InfoHash)
if readmePath := findReadme(opFlags.ReadmeRegex, metadata.Files); readmePath != nil {
completingCoordinator.Request(metadata.InfoHash, *readmePath, metadata.Peers)
}
case result := <-completingCoordinator.Output():
database.AddReadme(result.InfoHash, result.Path, result.Data)
case <-interruptChan:
trawlingManager.Terminate()
break
}
}
} }
func parseFlags() (opF opFlags) { func parseFlags() (opF opFlags) {
@ -237,13 +225,13 @@ func parseFlags() (opF opFlags) {
} }
if err = checkAddrs([]string{cmdF.LeechClAddr}); err != nil { if err = checkAddrs([]string{cmdF.LeechClAddr}); err != nil {
zap.S().Fatal("Of argument `leech-cl-addr` %s", err.Error()) zap.S().Fatalf("Of argument `leech-cl-addr` %s", err.Error())
} else { } else {
opF.LeechClAddr = cmdF.LeechClAddr opF.LeechClAddr = cmdF.LeechClAddr
} }
if err = checkAddrs([]string{cmdF.LeechMlAddr}); err != nil { if err = checkAddrs([]string{cmdF.LeechMlAddr}); err != nil {
zap.S().Fatal("Of argument `leech-ml-addr` %s", err.Error()) zap.S().Fatalf("Of argument `leech-ml-addr` %s", err.Error())
} else { } else {
opF.LeechMlAddr = cmdF.LeechMlAddr opF.LeechMlAddr = cmdF.LeechMlAddr
} }
@ -260,13 +248,9 @@ func parseFlags() (opF opFlags) {
opF.ReadmeMaxSize = cmdF.ReadmeMaxSize opF.ReadmeMaxSize = cmdF.ReadmeMaxSize
} }
for i, s := range cmdF.ReadmeRegexes { opF.ReadmeRegex, err = regexp.Compile(cmdF.ReadmeRegex)
regex, err := regexp.Compile(s) if err != nil {
if err != nil { zap.S().Fatalf("Argument `readme-regex` is not a valid regex: %s", err.Error())
zap.S().Fatalf("Of argument `readme-regex` with %d(th) regex `%s`: %s", i + 1, s, err.Error())
} else {
opF.ReadmeRegexes = append(opF.ReadmeRegexes, regex)
}
} }
opF.Verbosity = len(cmdF.Verbose) opF.Verbosity = len(cmdF.Verbose)
@ -286,3 +270,15 @@ func checkAddrs(addrs []string) error {
} }
return nil return nil
} }
// findReadme looks for a possible Readme file whose path is matched by the pathRegex.
// If there are multiple matches, the first one is returned.
// If there are no matches, nil returned.
func findReadme(pathRegex *regexp.Regexp, files []persistence.File) *string {
for _, file := range files {
if pathRegex.MatchString(file.Path) {
return &file.Path
}
}
return nil
}

113
src/magneticod/persistence.go → src/magneticod/persistence/database.go
View File

@ -1,4 +1,4 @@
package main package persistence
import ( import (
"bytes" "bytes"
@ -13,6 +13,7 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
"magneticod/bittorrent" "magneticod/bittorrent"
"regexp"
) )
type engineType uint8 type engineType uint8
@ -78,7 +79,7 @@ func NewDatabase(rawurl string) (*Database, error) {
} }
func (db *Database) DoesExist(infoHash []byte) bool { func (db *Database) DoesTorrentExist(infoHash []byte) bool {
for _, torrent := range db.newTorrents { for _, torrent := range db.newTorrents {
if bytes.Equal(infoHash, torrent.InfoHash) { if bytes.Equal(infoHash, torrent.InfoHash) {
return true; return true;
@ -96,6 +97,30 @@ func (db *Database) DoesExist(infoHash []byte) bool {
return rows.Next() return rows.Next()
} }
func (db *Database) FindAnIncompleteTorrent(pathRegex *regexp.Regexp, maxSize uint) error {
switch db.engine {
case SQLITE:
return db.findAnIncompleteTorrent_SQLite(pathRegex, maxSize)
default:
zap.L().Fatal("Unknown database engine!", zap.Uint8("engine", uint8(db.engine)))
return nil
}
}
func (db *Database) findAnIncompleteTorrent_SQLite(pathRegex *regexp.Regexp, maxSize uint) error {
// TODO: Prefer torrents with most seeders & leechs (i.e. most popular)
_, err := db.database.Query(`
SELECT torrents.info_hash, files.path FROM files WHERE files.path REGEXP ?
INNER JOIN torrents ON files.torrent_id = torrents.id LIMIT 1;
`)
if err != nil {
return err
}
return nil
}
// AddNewTorrent adds a new torrent to the *queue* to be flushed to the persistent database. // AddNewTorrent adds a new torrent to the *queue* to be flushed to the persistent database.
func (db *Database) AddNewTorrent(torrent bittorrent.Metadata) error { func (db *Database) AddNewTorrent(torrent bittorrent.Metadata) error {
@ -107,7 +132,7 @@ func (db *Database) AddNewTorrent(torrent bittorrent.Metadata) error {
// that it doesn't exists there, add it to the sink. // that it doesn't exists there, add it to the sink.
// Hence check for the last time whether the torrent exists in the database, and only if not, // Hence check for the last time whether the torrent exists in the database, and only if not,
// add it. // add it.
if db.DoesExist(torrent.InfoHash) { if db.DoesTorrentExist(torrent.InfoHash) {
return nil; return nil;
} }
@ -125,8 +150,14 @@ func (db *Database) AddNewTorrent(torrent bittorrent.Metadata) error {
} }
func (db *Database) AddReadme(infoHash []byte, path string, data []byte) error {
// TODO
return nil
}
func (db *Database) commitNewTorrents() error { func (db *Database) commitNewTorrents() error {
tx, err := db.database.Begin() tx, err := db.database.Begin()
if err != nil { if err != nil {
return fmt.Errorf("sql.DB.Begin()! %s", err.Error()) return fmt.Errorf("sql.DB.Begin()! %s", err.Error())
} }
@ -216,11 +247,11 @@ func setupSqliteDatabase(database *sql.DB) error {
// //
// Enable foreign key constraints in SQLite which are crucial to prevent programmer errors on // Enable foreign key constraints in SQLite which are crucial to prevent programmer errors on
// our side. // our side.
_, err := database.Exec( _, err := database.Exec(`
`PRAGMA journal_mode=WAL; PRAGMA journal_mode=WAL;
PRAGMA temp_store=1; PRAGMA temp_store=1;
PRAGMA foreign_keys=ON;`, PRAGMA foreign_keys=ON;
) `)
if err != nil { if err != nil {
return err return err
} }
@ -231,33 +262,28 @@ func setupSqliteDatabase(database *sql.DB) error {
} }
// Essential, and valid for all user_version`s: // Essential, and valid for all user_version`s:
_, err = tx.Exec( // TODO: "torrent_id" column of the "files" table can be NULL, how can we fix this in a new schema?
`CREATE TABLE IF NOT EXISTS torrents ( _, err = tx.Exec(`
CREATE TABLE IF NOT EXISTS torrents (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
info_hash BLOB NOT NULL UNIQUE, info_hash BLOB NOT NULL UNIQUE,
name TEXT NOT NULL, name TEXT NOT NULL,
total_size INTEGER NOT NULL CHECK(total_size > 0), total_size INTEGER NOT NULL CHECK(total_size > 0),
discovered_on INTEGER NOT NULL CHECK(discovered_on > 0) discovered_on INTEGER NOT NULL CHECK(discovered_on > 0)
); );
CREATE INDEX IF NOT EXISTS info_hash_index ON torrents (info_hash);
CREATE TABLE IF NOT EXISTS files ( CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
torrent_id INTEGER REFERENCES torrents ON DELETE CASCADE ON UPDATE RESTRICT, torrent_id INTEGER REFERENCES torrents ON DELETE CASCADE ON UPDATE RESTRICT,
size INTEGER NOT NULL, size INTEGER NOT NULL,
path TEXT NOT NULL path TEXT NOT NULL
); );
`, `)
)
if err != nil { if err != nil {
return err return err
} }
// Get the user_version: // Get the user_version:
res, err := tx.Query( res, err := tx.Query(`PRAGMA user_version;`)
`PRAGMA user_version;`,
)
if err != nil { if err != nil {
return err return err
} }
@ -265,18 +291,57 @@ func setupSqliteDatabase(database *sql.DB) error {
res.Next() res.Next()
res.Scan(&userVersion) res.Scan(&userVersion)
// Upgrade to the latest schema:
switch userVersion { switch userVersion {
// Upgrade from user_version 0 to 1 // Upgrade from user_version 0 to 1
// The Change:
// * `info_hash_index` is recreated as UNIQUE.
case 0: case 0:
_, err = tx.Exec( zap.S().Warnf("Updating database schema from 0 to 1... (this might take a while)")
`ALTER TABLE torrents ADD COLUMN readme TEXT; _, err = tx.Exec(`
PRAGMA user_version = 1;`, DROP INDEX info_hash_index;
) CREATE UNIQUE INDEX info_hash_index ON torrents (info_hash);
PRAGMA user_version = 1;
`)
if err != nil {
return err
}
fallthrough
// Upgrade from user_version 1 to 2
// The Change:
// * Added `is_readme` and `content` columns to the `files` table, and the constraints & the
// the indices they entail.
// * Added unique index `readme_index` on `files` table.
case 1:
zap.S().Warnf("Updating database schema from 1 to 2... (this might take a while)")
// We introduce two new columns here: content BLOB, and is_readme INTEGER which we treat as
// a bool (hence the CHECK).
// The reason for the change is that as we introduce the new "readme" feature which
// downloads a readme file as a torrent descriptor, we needed to store it somewhere in the
// database with the following conditions:
//
// 1. There can be one and only one readme (content) for a given torrent; hence the
// UNIQUE INDEX on (torrent_id, is_description) (remember that SQLite treats each NULL
// value as distinct [UNIQUE], see https://sqlite.org/nulls.html).
// 2. We would like to keep the readme (content) associated with the file it came from;
// hence we modify the files table instead of the torrents table.
//
// Regarding the implementation details, following constraints arise:
//
// 1. The column is_readme is either NULL or 1, and if it is 1, then content column cannot
// be NULL (but might be an empty BLOB). Vice versa, if content column of a row is,
// NULL then is_readme must be NULL.
//
// This is to prevent unused content fields filling up the database, and to catch
// programmers' errors.
_, err = tx.Exec(`
ALTER TABLE files ADD COLUMN is_readme INTEGER CHECK (is_readme IS NULL OR is_readme=1);
ALTER TABLE files ADD COLUMN content BLOB CHECK((content IS NULL AND is_readme IS NULL) OR (content IS NOT NULL AND is_readme=1));
CREATE UNIQUE INDEX readme_index ON files (torrent_id, is_readme);
PRAGMA user_version = 2;
`)
if err != nil { if err != nil {
return err return err
} }
// Add `fallthrough`s as needed to keep upgrading...
} }
if err = tx.Commit(); err != nil { if err = tx.Commit(); err != nil {

2
src/magneticod/persistence_test.go → src/magneticod/persistence/database_test.go
View File

@ -1,4 +1,4 @@
package main package persistence
import ( import (
"path" "path"

262
src/magneticow/bindata.go Normal file
View File

@ -0,0 +1,262 @@
package main
import (
"fmt"
"io/ioutil"
)
// bindata_read reads the given file from disk. It returns
// an error on failure.
func bindata_read(path, name string) ([]byte, error) {
buf, err := ioutil.ReadFile(path)
if err != nil {
err = fmt.Errorf("Error reading asset %s at %s: %v", name, path, err)
}
return buf, err
}
// templates_torrent_html reads file data from disk.
// It panics if something went wrong in the process.
func templates_torrent_html() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/templates/torrent.html",
"templates/torrent.html",
)
}
// templates_feed_xml reads file data from disk.
// It panics if something went wrong in the process.
func templates_feed_xml() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/templates/feed.xml",
"templates/feed.xml",
)
}
// templates_homepage_html reads file data from disk.
// It panics if something went wrong in the process.
func templates_homepage_html() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/templates/homepage.html",
"templates/homepage.html",
)
}
// templates_statistics_html reads file data from disk.
// It panics if something went wrong in the process.
func templates_statistics_html() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/templates/statistics.html",
"templates/statistics.html",
)
}
// templates_torrents_html reads file data from disk.
// It panics if something went wrong in the process.
func templates_torrents_html() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/templates/torrents.html",
"templates/torrents.html",
)
}
// static_scripts_plotly_v1_26_1_min_js reads file data from disk.
// It panics if something went wrong in the process.
func static_scripts_plotly_v1_26_1_min_js() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/scripts/plotly-v1.26.1.min.js",
"static/scripts/plotly-v1.26.1.min.js",
)
}
// static_scripts_statistics_js reads file data from disk.
// It panics if something went wrong in the process.
func static_scripts_statistics_js() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/scripts/statistics.js",
"static/scripts/statistics.js",
)
}
// static_scripts_torrent_js reads file data from disk.
// It panics if something went wrong in the process.
func static_scripts_torrent_js() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/scripts/torrent.js",
"static/scripts/torrent.js",
)
}
// static_styles_reset_css reads file data from disk.
// It panics if something went wrong in the process.
func static_styles_reset_css() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/styles/reset.css",
"static/styles/reset.css",
)
}
// static_styles_statistics_css reads file data from disk.
// It panics if something went wrong in the process.
func static_styles_statistics_css() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/styles/statistics.css",
"static/styles/statistics.css",
)
}
// static_styles_torrent_css reads file data from disk.
// It panics if something went wrong in the process.
func static_styles_torrent_css() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/styles/torrent.css",
"static/styles/torrent.css",
)
}
// static_styles_torrents_css reads file data from disk.
// It panics if something went wrong in the process.
func static_styles_torrents_css() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/styles/torrents.css",
"static/styles/torrents.css",
)
}
// static_styles_homepage_css reads file data from disk.
// It panics if something went wrong in the process.
func static_styles_homepage_css() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/styles/homepage.css",
"static/styles/homepage.css",
)
}
// static_styles_essential_css reads file data from disk.
// It panics if something went wrong in the process.
func static_styles_essential_css() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/styles/essential.css",
"static/styles/essential.css",
)
}
// static_assets_magnet_gif reads file data from disk.
// It panics if something went wrong in the process.
func static_assets_magnet_gif() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/assets/magnet.gif",
"static/assets/magnet.gif",
)
}
// static_assets_feed_png reads file data from disk.
// It panics if something went wrong in the process.
func static_assets_feed_png() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/assets/feed.png",
"static/assets/feed.png",
)
}
// static_fonts_notomono_license_ofl_txt reads file data from disk.
// It panics if something went wrong in the process.
func static_fonts_notomono_license_ofl_txt() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoMono/LICENSE_OFL.txt",
"static/fonts/NotoMono/LICENSE_OFL.txt",
)
}
// static_fonts_notomono_regular_ttf reads file data from disk.
// It panics if something went wrong in the process.
func static_fonts_notomono_regular_ttf() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoMono/Regular.ttf",
"static/fonts/NotoMono/Regular.ttf",
)
}
// static_fonts_notosansui_license_ofl_txt reads file data from disk.
// It panics if something went wrong in the process.
func static_fonts_notosansui_license_ofl_txt() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoSansUI/LICENSE_OFL.txt",
"static/fonts/NotoSansUI/LICENSE_OFL.txt",
)
}
// static_fonts_notosansui_bold_ttf reads file data from disk.
// It panics if something went wrong in the process.
func static_fonts_notosansui_bold_ttf() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoSansUI/Bold.ttf",
"static/fonts/NotoSansUI/Bold.ttf",
)
}
// static_fonts_notosansui_bolditalic_ttf reads file data from disk.
// It panics if something went wrong in the process.
func static_fonts_notosansui_bolditalic_ttf() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoSansUI/BoldItalic.ttf",
"static/fonts/NotoSansUI/BoldItalic.ttf",
)
}
// static_fonts_notosansui_italic_ttf reads file data from disk.
// It panics if something went wrong in the process.
func static_fonts_notosansui_italic_ttf() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoSansUI/Italic.ttf",
"static/fonts/NotoSansUI/Italic.ttf",
)
}
// static_fonts_notosansui_regular_ttf reads file data from disk.
// It panics if something went wrong in the process.
func static_fonts_notosansui_regular_ttf() ([]byte, error) {
return bindata_read(
"/home/bora/labs/magnetico/src/magneticow/data/static/fonts/NotoSansUI/Regular.ttf",
"static/fonts/NotoSansUI/Regular.ttf",
)
}
// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
if f, ok := _bindata[name]; ok {
return f()
}
return nil, fmt.Errorf("Asset %s not found", name)
}
// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string] func() ([]byte, error) {
"templates/torrent.html": templates_torrent_html,
"templates/feed.xml": templates_feed_xml,
"templates/homepage.html": templates_homepage_html,
"templates/statistics.html": templates_statistics_html,
"templates/torrents.html": templates_torrents_html,
"static/scripts/plotly-v1.26.1.min.js": static_scripts_plotly_v1_26_1_min_js,
"static/scripts/statistics.js": static_scripts_statistics_js,
"static/scripts/torrent.js": static_scripts_torrent_js,
"static/styles/reset.css": static_styles_reset_css,
"static/styles/statistics.css": static_styles_statistics_css,
"static/styles/torrent.css": static_styles_torrent_css,
"static/styles/torrents.css": static_styles_torrents_css,
"static/styles/homepage.css": static_styles_homepage_css,
"static/styles/essential.css": static_styles_essential_css,
"static/assets/magnet.gif": static_assets_magnet_gif,
"static/assets/feed.png": static_assets_feed_png,
"static/fonts/NotoMono/LICENSE_OFL.txt": static_fonts_notomono_license_ofl_txt,
"static/fonts/NotoMono/Regular.ttf": static_fonts_notomono_regular_ttf,
"static/fonts/NotoSansUI/LICENSE_OFL.txt": static_fonts_notosansui_license_ofl_txt,
"static/fonts/NotoSansUI/Bold.ttf": static_fonts_notosansui_bold_ttf,
"static/fonts/NotoSansUI/BoldItalic.ttf": static_fonts_notosansui_bolditalic_ttf,
"static/fonts/NotoSansUI/Italic.ttf": static_fonts_notosansui_italic_ttf,
"static/fonts/NotoSansUI/Regular.ttf": static_fonts_notosansui_regular_ttf,
}

0
magneticow/magneticow/static/assets/feed.png → src/magneticow/data/static/assets/feed.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 531 B

After

Width:  |  Height:  |  Size: 531 B

0
magneticow/magneticow/static/assets/magnet.gif → src/magneticow/data/static/assets/magnet.gif
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 148 B

After

Width:  |  Height:  |  Size: 148 B

0
magneticow/magneticow/static/fonts/NotoMono/LICENSE_OFL.txt → src/magneticow/data/static/fonts/NotoMono/LICENSE_OFL.txt
View File

0
magneticow/magneticow/static/fonts/NotoMono/Regular.ttf → src/magneticow/data/static/fonts/NotoMono/Regular.ttf
View File

0
magneticow/magneticow/static/fonts/NotoSansUI/Bold.ttf → src/magneticow/data/static/fonts/NotoSansUI/Bold.ttf
View File

0
magneticow/magneticow/static/fonts/NotoSansUI/BoldItalic.ttf → src/magneticow/data/static/fonts/NotoSansUI/BoldItalic.ttf
View File

0
magneticow/magneticow/static/fonts/NotoSansUI/Italic.ttf → src/magneticow/data/static/fonts/NotoSansUI/Italic.ttf
View File

0
magneticow/magneticow/static/fonts/NotoSansUI/LICENSE_OFL.txt → src/magneticow/data/static/fonts/NotoSansUI/LICENSE_OFL.txt
View File

0
magneticow/magneticow/static/fonts/NotoSansUI/Regular.ttf → src/magneticow/data/static/fonts/NotoSansUI/Regular.ttf
View File

0
magneticow/magneticow/static/scripts/plotly-v1.26.1.min.js → src/magneticow/data/static/scripts/plotly-v1.26.1.min.js vendored
View File

0
magneticow/magneticow/static/scripts/statistics.js → src/magneticow/data/static/scripts/statistics.js
View File

0
magneticow/magneticow/static/scripts/torrent.js → src/magneticow/data/static/scripts/torrent.js
View File

8
magneticow/magneticow/static/styles/essential.css → src/magneticow/data/static/styles/essential.css
View File

@ -39,7 +39,7 @@ html {
} }
pre { pre {
font-family: 'Noto Mono'; font-family: 'Noto Mono', monospace;
line-height: 1.2em; line-height: 1.2em;
} }
@ -49,6 +49,12 @@ body {
line-height: 1.45; line-height: 1.45;
} }
@media (max-width: 616px) {
body {
padding: 1em 8px 1em 8px;
}
}
b { b {
font-weight: bold; font-weight: bold;
} }

38
src/magneticow/data/static/styles/homepage.css Normal file
View File

@ -0,0 +1,38 @@
main {
display: flex;
align-items: center;
align-content: center;
height: calc(100vh - 2*16px - 0.833em - 23px); /* 100vh - body's padding(s) - footer margin - footer height */
width: 100%;
}
@media (max-width: 616px) {
main {
flex-direction: column;
justify-content: center;
align-items: flex-start;
}
}
main div#magneticow {
white-space: nowrap;
margin: 0 0.5em 0.5em 0;
}
main form {
max-width: 600px;
width: 100%;
}
main form input {
width: 100%;
}
main > div {
margin-right: 0.5em;
}
footer {
margin-top: 0.833em;
}

0
magneticow/magneticow/static/styles/reset.css → src/magneticow/data/static/styles/reset.css
View File

0
magneticow/magneticow/static/styles/statistics.css → src/magneticow/data/static/styles/statistics.css
View File

0
magneticow/magneticow/static/styles/torrent.css → src/magneticow/data/static/styles/torrent.css
View File

0
magneticow/magneticow/static/styles/torrents.css → src/magneticow/data/static/styles/torrents.css
View File

5
magneticow/magneticow/templates/feed.xml → src/magneticow/data/templates/feed.xml
View File

@ -1,12 +1,15 @@
<?xml version="1.0" encoding="utf-8" standalone="yes"?> <?xml version="1.0" encoding="utf-8" standalone="yes"?>
<rss version="2.0"> <rss version="2.0">
<channel> <channel>
<title>{{ title }}</title> <title>{{ .Title }}</title>
{% for item in items %} {% for item in items %}
<item> <item>
<title>{{ item.title }}</title> <title>{{ item.title }}</title>
<pubDate>{{ item.DiscoveredOn }}</pubDate>
<guid>{{ item.info_hash }}</guid> <guid>{{ item.info_hash }}</guid>
<enclosure url="magnet:?xt=urn:btih:{{ item.info_hash }}&amp;dn={{ item.title }}" type="application/x-bittorrent" /> <enclosure url="magnet:?xt=urn:btih:{{ item.info_hash }}&amp;dn={{ item.title }}" type="application/x-bittorrent" />
<description><![CDATA[Seeders: {{ item.NSeeders }} - Leechers: {{ item.NLeechers }}]]></description>
</item> </item>
{% endfor %} {% endfor %}
</channel> </channel>

24
src/magneticow/data/templates/homepage.html Normal file
View File

@ -0,0 +1,24 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>magneticow</title>
<link rel="stylesheet" href="static/styles/reset.css">
<link rel="stylesheet" href="static/styles/essential.css">
<link rel="stylesheet" href="static/styles/homepage.css">
<!-- <script src="script.js"></script> -->
</head>
<body>
<main>
<div id="magneticow"><b>magnetico<sup>w</sup></b>&#8203;<sub>(pre-alpha)</sub></div>
<form action="/torrents" method="get" autocomplete="off" role="search">
<input type="search" name="search" placeholder="Search the BitTorrent DHT" autofocus>
</form>
</main>
<footer>
~{{ . }} torrents available (see the <a href="/statistics">statistics</a>).
</footer>
</body>
</html>

0
magneticow/magneticow/templates/statistics.html → src/magneticow/data/templates/statistics.html
View File

0
magneticow/magneticow/templates/torrent.html → src/magneticow/data/templates/torrent.html
View File

66
src/magneticow/data/templates/torrents.html Normal file
View File

@ -0,0 +1,66 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{% if .search %}"{{.search}}"{% else %}Most recent torrents{% endif %} - magneticow</title>
<link rel="stylesheet" href="static/styles/reset.css">
<link rel="stylesheet" href="static/styles/essential.css">
<link rel="stylesheet" href="static/styles/torrents.css">
<!-- <script src="script.js"></script> -->
</head>
<body>
<header>
<div><a href="/"><b>magnetico<sup>w</sup></b></a>&#8203;<sub>(pre-alpha)</sub></div>
<form action="/torrents" method="get" autocomplete="off" role="search">
<input type="search" name="search" placeholder="Search the BitTorrent DHT" value="{{ .search }}">
</form>
<div>
<a href="{{ .subscription_url }}"><img src="static/assets/feed.png"
alt="feed icon" title="subscribe" /> subscribe</a>
</div>
</header>
<main>
<table>
<thead>
<tr>
<th><!-- Magnet link --></th>
<th>Name</th>
<th>Size</th>
<th>Discovered on</th>
</tr>
</thead>
<tbody>
{% for torrent in torrents %}
<tr>
<td><a href="magnet:?xt=urn:btih:{{ .torrent.info_hash }}&dn={{ .torrent.name }}">
<img src="static/assets/magnet.gif" alt="Magnet link"
title="Download this torrent using magnet" /></a></td>
<td><a href="/torrents/{{ .torrent.info_hash }}/{{ .torrent.name }}">{{ torrent.name }}</a></td>
<td>{{ torrent.size }}</td>
<td>{{ torrent.discovered_on }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</main>
<footer>
<form action="/torrents" method="get">
<button {% if page == 0 %}disabled{% endif %}>Previous</button>
<input type="text" name="search" value="{{ search }}" hidden>
{% if sorted_by %}
<input type="text" name="sort_by" value="{{ sorted_by }}" hidden>
{% endif %}
<input type="number" name="page" value="{{ page - 1 }}" hidden>
</form>
<form action="/torrents" method="get">
<button {% if not next_page_exists %}disabled{% endif %}>Next</button>
<input type="text" name="search" value="{{ search }}" hidden>
{% if sorted_by %}
<input type="text" name="sort_by" value="{{ sorted_by }}" hidden>
{% endif %}
<input type="number" name="page" value="{{ page + 1 }}" hidden>
</form>
</footer>
</body>
</html>

81
src/magneticow/main.go
View File

@ -1,25 +1,20 @@
// magneticow - Lightweight web interface for magnetico.
// Copyright (C) 2017 Mert Bora ALPER <bora@boramalper.org>
// Dedicated to Cemile Binay, in whose hands I thrived.
//
// This program is free software: you can redistribute it and/or modify it under the terms of the
// GNU General Public License as published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
// even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public License along with this program. If
// not, see <http://www.gnu.org/licenses/>.
package main package main
import ( import (
"html/template"
"log"
"net/http" "net/http"
"strings"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"persistence"
) )
const N_TORRENTS = 20
var templates map[string]*template.Template
var database persistence.Database
func main() { func main() {
router := mux.NewRouter() router := mux.NewRouter()
@ -28,23 +23,48 @@ func main() {
router.HandleFunc("/torrents/{infohash}", torrentsInfohashHandler) router.HandleFunc("/torrents/{infohash}", torrentsInfohashHandler)
router.HandleFunc("/torrents/{infohash}/{name}", torrentsInfohashNameHandler) router.HandleFunc("/torrents/{infohash}/{name}", torrentsInfohashNameHandler)
router.HandleFunc("/statistics", statisticsHandler) router.HandleFunc("/statistics", statisticsHandler)
router.PathPrefix("/static").HandlerFunc(staticHandler)
router.HandleFunc("/feed", feedHandler) router.HandleFunc("/feed", feedHandler)
templates = make(map[string]*template.Template)
templates["feed"] = template.Must(template.New("feed").Parse(string(mustAsset("templates/feed.xml"))))
templates["homepage"] = template.Must(template.New("homepage").Parse(string(mustAsset("templates/homepage.html"))))
templates["statistics"] = template.Must(template.New("statistics").Parse(string(mustAsset("templates/statistics.html"))))
templates["torrent"] = template.Must(template.New("torrent").Parse(string(mustAsset("templates/torrent.html"))))
templates["torrents"] = template.Must(template.New("torrents").Parse(string(mustAsset("templates/torrents.html"))))
var err error
database, err = persistence.MakeDatabase("sqlite3:///home/bora/.local/share/magneticod/database.sqlite3")
if err != nil {
panic(err.Error())
}
http.ListenAndServe(":8080", router) http.ListenAndServe(":8080", router)
} }
func rootHandler(w http.ResponseWriter, r *http.Request) { func rootHandler(w http.ResponseWriter, r *http.Request) {
count, err := database.GetNumberOfTorrents()
if err != nil {
panic(err.Error())
}
templates["homepage"].Execute(w, count)
} }
func torrentsHandler(w http.ResponseWriter, r *http.Request) { func torrentsHandler(w http.ResponseWriter, r *http.Request) {
/*
newestTorrents, err := database.NewestTorrents(N_TORRENTS)
if err != nil {
panic(err.Error())
}
templates["torrents"].Execute(w, nil)
*/
} }
func torrentsInfohashHandler(w http.ResponseWriter, r *http.Request) { func torrentsInfohashHandler(w http.ResponseWriter, r *http.Request) {
// redirect to torrents/{infohash}/name
} }
@ -61,3 +81,28 @@ func statisticsHandler(w http.ResponseWriter, r *http.Request) {
func feedHandler(w http.ResponseWriter, r *http.Request) { func feedHandler(w http.ResponseWriter, r *http.Request) {
} }
func staticHandler(w http.ResponseWriter, r *http.Request) {
data, err := Asset(r.URL.Path[1:])
if err != nil {
http.NotFound(w, r)
return
}
var contentType string
if strings.HasSuffix(r.URL.Path, ".css") {
contentType = "text/css; charset=utf-8"
} else { // fallback option
contentType = http.DetectContentType(data)
}
w.Header().Set("Content-Type", contentType)
w.Write(data)
}
func mustAsset(name string) []byte {
data, err := Asset(name)
if err != nil {
log.Panicf("Could NOT access the requested resource `%s`: %s", name, err.Error())
}
return data
}

26
src/persistence/Gopkg.toml Normal file
View File

@ -0,0 +1,26 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
[[constraint]]
name = "go.uber.org/zap"
version = "1.6.0"

116
src/persistence/interface.go Normal file
View File

@ -0,0 +1,116 @@
package persistence
import (
"fmt"
"regexp"
"net/url"
)
type Database interface {
Engine() databaseEngine
DoesTorrentExist(infoHash []byte) (bool, error)
// GiveAnIncompleteTorrentByInfoHash returns (*gives*) an incomplete -i.e. one that doesn't have
// readme downloaded yet- torrent from the database.
// GiveAnIncompleteTorrent might return a nil slice for infoHash, a nil string, and a nil err,
// meaning that no incomplete torrent could be found in the database (congrats!).
GiveAnIncompleteTorrent(pathRegex *regexp.Regexp, maxSize uint) (infoHash []byte, path string, err error)
GiveAStaleTorrent() (infoHash []byte, err error)
AddNewTorrent(infoHash []byte, name string, files []File) error
AddReadme(infoHash []byte, path string, content string) error
Close() error
// GetNumberOfTorrents returns the number of torrents saved in the database. Might be an
// approximation.
GetNumberOfTorrents() (uint, error)
NewestTorrents(n uint) ([]TorrentMetadata, error)
SearchTorrents(query string, orderBy orderingCriteria, descending bool, mustHaveReadme bool) ([]TorrentMetadata, error)
// GetTorrents returns the TorrentExtMetadata for the torrent of the given infoHash. Might return
// nil, nil if the torrent does not exist in the database.
GetTorrent(infoHash []byte) (*TorrentMetadata, error)
GetFiles(infoHash []byte) ([]File, error)
GetReadme(infoHash []byte) (string, error)
GetStatistics(from ISO8601, period uint) (*Statistics, error)
}
type orderingCriteria uint8
const (
BY_NAME orderingCriteria = 1
BY_SIZE = 2
BY_DISCOVERED_ON = 3
BY_N_FILES = 4
BY_N_SEEDERS = 5
BY_N_LEECHERS = 6
BY_UPDATED_ON = 7
BY_N_SEEDERS_TO_N_LEECHERS_RATIO = 8
BY_N_SEEDERS_PLUS_N_LEECHERS = 9
)
type statisticsGranularity uint8
type ISO8601 string
const (
MINUTELY_STATISTICS statisticsGranularity = 1
HOURLY_STATISTICS = 2
DAILY_STATISTICS = 3
WEEKLY_STATISTICS = 4
MONTHLY_STATISTICS = 5
YEARLY_STATISTICS = 6
)
type databaseEngine uint8
const (
SQLITE3_ENGINE databaseEngine = 1
)
type Statistics struct {
Granularity statisticsGranularity
From ISO8601
Period uint
// All these slices below have the exact length equal to the Period.
NTorrentsDiscovered []uint
NFilesDiscovered []uint
NReadmesDownloaded []uint
NTorrentsUpdated []uint
}
type File struct {
Size int64
Path string
}
type TorrentMetadata struct {
infoHash []byte
name string
size uint64
discoveredOn int64
hasReadme bool
nFiles uint
// values below 0 indicates that no data is available:
nSeeders int
nLeechers int
updatedOn int
}
func MakeDatabase(rawURL string) (Database, error) {
url_, err := url.Parse(rawURL)
if err != nil {
return nil, err
}
switch url_.Scheme {
case "sqlite3":
return makeSqlite3Database(url_)
case "postgresql":
return nil, fmt.Errorf("postgresql is not yet supported!")
case "mysql":
return nil, fmt.Errorf("mysql is not yet supported!")
}
return nil, fmt.Errorf("unknown URI scheme (database engine)!")
}

1
src/persistence/mysql.go Normal file
View File

@ -0,0 +1 @@
package persistence

1
src/persistence/postgresql.go Normal file
View File

@ -0,0 +1 @@
package persistence

427
src/persistence/sqlite3.go Normal file
View File

@ -0,0 +1,427 @@
package persistence
import (
"net/url"
"path"
"os"
"fmt"
"database/sql"
"regexp"
"go.uber.org/zap"
"time"
)
type sqlite3Database struct {
conn *sql.DB
}
func (db *sqlite3Database) Engine() databaseEngine {
return SQLITE3_ENGINE
}
func makeSqlite3Database(url_ *url.URL) (Database, error) {
db := new(sqlite3Database)
dbDir, _ := path.Split(url_.Path)
if err := os.MkdirAll(dbDir, 0755); err != nil {
return nil, fmt.Errorf("for directory `%s`: %s", dbDir, err.Error())
}
var err error
db.conn, err = sql.Open("sqlite3", url_.Path)
if err != nil {
return nil, err
}
// > Open may just validate its arguments without creating a connection to the database. To
// > verify that the data source name is valid, call Ping.
// https://golang.org/pkg/database/sql/#Open
if err = db.conn.Ping(); err != nil {
return nil, err
}
if err := db.setupDatabase(); err != nil {
return nil, err
}
return db, nil
}
func (db *sqlite3Database) DoesTorrentExist(infoHash []byte) (bool, error) {
rows, err := db.conn.Query("SELECT 1 FROM torrents WHERE info_hash = ?;", infoHash)
if err != nil {
return false, err;
}
// If rows.Next() returns true, meaning that the torrent is in the database, return true; else
// return false.
exists := rows.Next()
if err = rows.Close(); err != nil {
return false, err;
}
return exists, nil;
}
func (db *sqlite3Database) GiveAnIncompleteTorrent(pathRegex *regexp.Regexp, maxSize uint) (infoHash []byte, path string, err error) {
rows, err := db.conn.Query("SELECT info_hash FROM torrents WHERE has_readme = 0;")
if err != nil {
return nil, "", err
}
if rows.Next() != true {
return nil, "", nil
}
if err = rows.Scan(&infoHash); err != nil {
return nil, "", err
}
if err = rows.Close(); err != nil {
return nil, "", err
}
// TODO
return infoHash, "", nil
}
func (db *sqlite3Database) GiveAStaleTorrent() (infoHash []byte, err error) {
// TODO
return nil, nil
}
func (db *sqlite3Database) AddNewTorrent(infoHash []byte, name string, files []File) error {
// Although we check whether the torrent exists in the database before asking MetadataSink to
// fetch its metadata, the torrent can also exists in the Sink before that. Now, if a torrent in
// the sink is still being fetched, that's still not a problem as we just add the new peer for
// the torrent and exit, but if the torrent is complete (i.e. its metadata) and if its waiting
// in the channel to be received, a race condition arises when we query the database and seeing
// that it doesn't exists there, add it to the sink.
// Hence check for the last time whether the torrent exists in the database, and only if not,
// add it.
exists, err := db.DoesTorrentExist(infoHash)
if err != nil {
return err;
} else if exists {
return nil;
}
tx, err := db.conn.Begin()
if err != nil {
return err
}
// If everything goes as planned and no error occurs, we will commit the transaction before
// returning from the function so the tx.Rollback() call will fail, trying to rollback a
// committed transaction. BUT, if an error occurs, we'll get our transaction rollback'ed, which
// is nice.
defer tx.Rollback()
var total_size int64 = 0
for _, file := range files {
total_size += file.Size
}
res, err := tx.Exec(`
INSERT INTO torrents (
info_hash,
name,
total_size,
discovered_on,
n_files,
) VALUES (?, ?, ?, ?, ?, ?, ?, ?);
`, infoHash, name, total_size, time.Now().Unix(), len(files))
if err != nil {
return err
}
var lastInsertId int64
if lastInsertId, err = res.LastInsertId(); err != nil {
return fmt.Errorf("sql.Result.LastInsertId()! %s", err.Error())
}
for _, file := range files {
_, err = tx.Exec("INSERT INTO files (torrent_id, size, path) VALUES (?, ?, ?);",
lastInsertId, file.Size, file.Path,
)
if err != nil {
return err
}
}
err = tx.Commit()
if err != nil {
return err
}
return nil
}
func (db *sqlite3Database) AddReadme(infoHash []byte, path string, content string) error {
_, err := db.conn.Exec(
`UPDATE files SET is_readme = 1, content = ?
WHERE path = ? AND (SELECT id FROM torrents WHERE info_hash = ?) = torrent_id;`,
content, path, infoHash,
)
if err != nil {
return err
}
return nil
}
func (db *sqlite3Database) Close() error {
return db.conn.Close()
}
func (db *sqlite3Database) GetNumberOfTorrents() (uint, error) {
// COUNT(ROWID) is much more inefficient since it scans the whole table, so use MAX(ROWID)
rows, err := db.conn.Query("SELECT MAX(ROWID) FROM torrents;")
if err != nil {
return 0, err
}
if rows.Next() != true {
fmt.Errorf("No rows returned from `SELECT MAX(ROWID)`!")
}
var n uint
if err = rows.Scan(&n); err != nil {
return 0, err
}
if err = rows.Close(); err != nil {
return 0, err
}
return n, nil
}
func (db *sqlite3Database) NewestTorrents(n uint) ([]TorrentMetadata, error) {
rows, err := db.conn.Query(`
SELECT
info_hash,
name,
total_size,
discovered_on,
has_readme,
n_files,
n_seeders,
n_leechers,
updated_on
FROM torrents
ORDER BY discovered_on DESC LIMIT ?;
`, n,
)
if err != nil {
return nil, err
}
var torrents []TorrentMetadata
for rows.Next() {
tm := new(TorrentMetadata)
rows.Scan(
&tm.infoHash, &tm.name, &tm.discoveredOn, &tm.hasReadme, &tm.nFiles, &tm.nSeeders,
&tm.nLeechers, &tm.updatedOn,
)
torrents = append(torrents, *tm)
}
if err = rows.Close(); err != nil {
return nil, err
}
return torrents, nil
}
func (db *sqlite3Database) SearchTorrents(query string, orderBy orderingCriteria, descending bool, mustHaveReadme bool) ([]TorrentMetadata, error) { // TODO
// TODO:
return nil, nil
}
func (db *sqlite3Database) GetTorrent(infoHash []byte) (*TorrentMetadata, error) {
rows, err := db.conn.Query(
`SELECT
info_hash,
name,
size,
discovered_on,
has_readme,
n_files,
n_seeders,
n_leechers,
updated_on
FROM torrents
WHERE info_hash = ?`,
infoHash,
)
if err != nil {
return nil, err
}
if rows.Next() != true {
return nil, nil
}
tm := new(TorrentMetadata)
rows.Scan(
&tm.infoHash, &tm.name, &tm.discoveredOn, &tm.hasReadme, &tm.nFiles, &tm.nSeeders,
&tm.nLeechers, &tm.updatedOn,
)
if err = rows.Close(); err != nil {
return nil, err
}
return tm, nil
}
func (db *sqlite3Database) GetFiles(infoHash []byte) ([]File, error) {
// TODO
return nil, nil
}
func (db *sqlite3Database) GetReadme(infoHash []byte) (string, error) {
// TODO
return "", nil
}
func (db *sqlite3Database) GetStatistics(from ISO8601, period uint) (*Statistics, error) {
// TODO
return nil, nil
}
func (db *sqlite3Database) commitQueuedTorrents() error {
return nil
}
func (db *sqlite3Database) setupDatabase() error {
// Enable Write-Ahead Logging for SQLite as "WAL provides more concurrency as readers do not
// block writers and a writer does not block readers. Reading and writing can proceed
// concurrently."
// Caveats:
// * Might be unsupported by OSes other than Windows and UNIXes.
// * Does not work over a network filesystem.
// * Transactions that involve changes against multiple ATTACHed databases are not atomic
// across all databases as a set.
// See: https://www.sqlite.org/wal.html
//
// Force SQLite to use disk, instead of memory, for all temporary files to reduce the memory
// footprint.
//
// Enable foreign key constraints in SQLite which are crucial to prevent programmer errors on
// our side.
_, err := db.conn.Exec(`
PRAGMA journal_mode=WAL;
PRAGMA temp_store=1;
PRAGMA foreign_keys=ON;
`)
if err != nil {
return err
}
tx, err := db.conn.Begin()
if err != nil {
return err
}
// If everything goes as planned and no error occurs, we will commit the transaction before
// returning from the function so the tx.Rollback() call will fail, trying to rollback a
// committed transaction. BUT, if an error occurs, we'll get our transaction rollback'ed, which
// is nice.
defer tx.Rollback()
// Essential, and valid for all user_version`s:
// TODO: "torrent_id" column of the "files" table can be NULL, how can we fix this in a new schema?
_, err = tx.Exec(`
CREATE TABLE IF NOT EXISTS torrents (
id INTEGER PRIMARY KEY,
info_hash BLOB NOT NULL UNIQUE,
name TEXT NOT NULL,
total_size INTEGER NOT NULL CHECK(total_size > 0),
discovered_on INTEGER NOT NULL CHECK(discovered_on > 0)
);
CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY,
torrent_id INTEGER REFERENCES torrents ON DELETE CASCADE ON UPDATE RESTRICT,
size INTEGER NOT NULL,
path TEXT NOT NULL
);
`)
if err != nil {
return err
}
// Get the user_version:
res, err := tx.Query("PRAGMA user_version;")
if err != nil {
return err
}
var userVersion int;
if res.Next() != true {
return fmt.Errorf("PRAGMA user_version did not return any rows!")
}
if err = res.Scan(&userVersion); err != nil {
return err
}
switch userVersion {
// Upgrade from user_version 0 to 1
// The Change:
// * `info_hash_index` is recreated as UNIQUE.
case 0:
zap.S().Warnf("Updating database schema from 0 to 1... (this might take a while)")
_, err = tx.Exec(`
DROP INDEX info_hash_index;
CREATE UNIQUE INDEX info_hash_index ON torrents (info_hash);
PRAGMA user_version = 1;
`)
if err != nil {
return err
}
fallthrough
// Upgrade from user_version 1 to 2
// The Change:
// * Added `is_readme` and `content` columns to the `files` table, and the constraints & the
// the indices they entail.
// * Added unique index `readme_index` on `files` table.
case 1:
zap.S().Warnf("Updating database schema from 1 to 2... (this might take a while)")
// We introduce two new columns here: content BLOB, and is_readme INTEGER which we treat as
// a bool (hence the CHECK).
// The reason for the change is that as we introduce the new "readme" feature which
// downloads a readme file as a torrent descriptor, we needed to store it somewhere in the
// database with the following conditions:
//
// 1. There can be one and only one readme (content) for a given torrent; hence the
// UNIQUE INDEX on (torrent_id, is_description) (remember that SQLite treats each NULL
// value as distinct [UNIQUE], see https://sqlite.org/nulls.html).
// 2. We would like to keep the readme (content) associated with the file it came from;
// hence we modify the files table instead of the torrents table.
//
// Regarding the implementation details, following constraints arise:
//
// 1. The column is_readme is either NULL or 1, and if it is 1, then content column cannot
// be NULL (but might be an empty BLOB). Vice versa, if content column of a row is,
// NULL then is_readme must be NULL.
//
// This is to prevent unused content fields filling up the database, and to catch
// programmers' errors.
_, err = tx.Exec(`
ALTER TABLE files ADD COLUMN is_readme INTEGER CHECK (is_readme IS NULL OR is_readme=1) DEFAULT NULL;
ALTER TABLE files ADD COLUMN content BLOB CHECK((content IS NULL AND is_readme IS NULL) OR (content IS NOT NULL AND is_readme=1)) DEFAULT NULL;
CREATE UNIQUE INDEX readme_index ON files (torrent_id, is_readme);
PRAGMA user_version = 2;
`)
if err != nil {
return err
}
}
if err = tx.Commit(); err != nil {
return err
}
return nil
}