[magneticow] the search now works, but need to change our approach

This commit is contained in:
Bora Alper 2018-04-25 21:33:50 +01:00
parent ac7d0a514f
commit 0c54cc80dc
8 changed files with 145 additions and 214 deletions

View File

@ -5,11 +5,11 @@ import (
"github.com/anacrolix/torrent/bencode"
"go.uber.org/zap"
"strings"
"golang.org/x/sys/unix"
)
type Transport struct {
conn *net.UDPConn
fd int
laddr *net.UDPAddr
started bool
@ -46,16 +46,18 @@ func (t *Transport) Start() {
t.started = true
var err error
t.conn, err = net.ListenUDP("udp", t.laddr)
t.fd, err = unix.Socket(unix.SOCK_DGRAM, unix.AF_INET, 0)
if err != nil {
zap.L().Fatal("Could NOT create a UDP socket!", zap.Error(err))
}
unix.Bind(t.fd, unix.SockaddrInet4{Addr: t.laddr.IP, Port: t.laddr.Port})
go t.readMessages()
}
func (t *Transport) Terminate() {
t.conn.Close()
unix.Close(t.fd);
}
// readMessages is a goroutine!
@ -63,15 +65,11 @@ func (t *Transport) readMessages() {
buffer := make([]byte, 65536)
for {
n, addr, err := t.conn.ReadFrom(buffer)
n, from, err := unix.Recvfrom(t.fd, buffer, 0)
if err != nil {
// TODO: isn't there a more reliable way to detect if UDPConn is closed?
if strings.HasSuffix(err.Error(), "use of closed network connection") {
break
} else {
zap.L().Debug("Could NOT read an UDP packet!", zap.Error(err))
}
}
var msg Message
err = bencode.Unmarshal(buffer[:n], &msg)
@ -79,7 +77,7 @@ func (t *Transport) readMessages() {
zap.L().Debug("Could NOT unmarshal packet data!", zap.Error(err))
}
t.onMessage(&msg, addr)
t.onMessage(&msg, from)
}
}
@ -89,9 +87,9 @@ func (t *Transport) WriteMessages(msg *Message, addr net.Addr) {
zap.L().Panic("Could NOT marshal an outgoing message! (Programmer error.)")
}
_, err = t.conn.WriteTo(data, addr)
err = unix.Sendto(t.fd, data, 0, addr)
// TODO: isn't there a more reliable way to detect if UDPConn is closed?
if err != nil && !strings.HasSuffix(err.Error(), "use of closed network connection") {
if err != nil {
zap.L().Debug("Could NOT write an UDP packet!", zap.Error(err))
}
}

View File

@ -0,0 +1,4 @@
function loadMore() {
console.log("lastX", canLoadMore, lastID, lastOrderedValue);
}

View File

@ -12,12 +12,12 @@
<main>
<div><b>magnetico<sup>w</sup></b>&#8203;<sub>(pre-alpha)</sub></div>
<form action="/torrents" method="get" autocomplete="off" role="search">
<input type="search" name="search" placeholder="Search the BitTorrent DHT" autofocus>
<input type="search" name="query" placeholder="Search the BitTorrent DHT" autofocus>
</form>
</main>
<footer>
~{{ "{:,}".format(n_torrents) }} torrents available (see the <a href="/statistics">statistics</a>).
~{{ comma .NTorrents }} torrents available (see the <a href="/statistics">statistics</a>).
</footer>
</body>
</html>

View File

@ -12,7 +12,7 @@
<header>
<div><a href="/"><b>magnetico<sup>w</sup></b></a>&#8203;<sub>(pre-alpha)</sub></div>
<form action="/torrents" method="get" autocomplete="off" role="search">
<input type="search" name="search" placeholder="Search the BitTorrent DHT">
<input type="search" name="query" placeholder="Search the BitTorrent DHT">
</form>
</header>
<main>

View File

@ -2,20 +2,25 @@
<html lang="en">
<head>
<meta charset="utf-8">
<title>{% if search %}"{{search}}"{% else %}Most recent torrents{% endif %} - magneticow</title>
<title>{{ if .Query }}"{{ .Query }}"{{ else }}Most recent torrents{{ end }} - magneticow</title>
<link rel="stylesheet" href="static/styles/reset.css">
<link rel="stylesheet" href="static/styles/essential.css">
<link rel="stylesheet" href="static/styles/torrents.css">
<!-- <script src="script.js"></script> -->
<script>
var canLoadMore = {{ if .CanLoadMore }} true {{ else }} false {{ end }};
var lastOrderedValue = {{ .LastOrderedValue }};
var lastID = {{ .LastID }};
</script>
<script src="static/scripts/torrents.js"></script>
</head>
<body>
<header>
<div><a href="/"><b>magnetico<sup>w</sup></b></a>&#8203;<sub>(pre-alpha)</sub></div>
<form action="/torrents" method="get" autocomplete="off" role="search">
<input type="search" name="search" placeholder="Search the BitTorrent DHT" value="{{ search }}">
<input type="search" name="query" placeholder="Search the BitTorrent DHT" value="{{ .Query }}">
</form>
<div>
<a href="{{ subscription_url }}"><img src="static/assets/feed.png"
<a href="{{ .SubscriptionURL }}"><img src="static/assets/feed.png"
alt="feed icon" title="subscribe" /> subscribe</a>
</div>
</header>
@ -24,67 +29,29 @@
<thead>
<tr>
<th><!-- Magnet link --></th>
<th>
{% if sorted_by == "name ASC" %}
<a href="/torrents/?search={{ search }}&sort_by=name+DESC">Name ▲</a>
{% elif sorted_by == "name DESC" %}
<a href="/torrents/?search={{ search }}&sort_by=name+ASC">Name ▼</a>
{% else %}
<a href="/torrents/?search={{ search }}&sort_by=name+ASC">Name</a>
{% endif %}
</th>
<th>
{% if sorted_by == "total_size ASC" %}
<a href="/torrents/?search={{ search }}&sort_by=total_size+DESC">Size ▲</a>
{% elif sorted_by == "total_size DESC" %}
<a href="/torrents/?search={{ search }}&sort_by=total_size+ASC">Size ▼</a>
{% else %}
<a href="/torrents/?search={{ search }}&sort_by=total_size+ASC">Size</a>
{% endif %}
</th>
<th>
{% if sorted_by == "discovered_on ASC" %}
<a href="/torrents/?search={{ search }}&sort_by=discovered_on+DESC">Discovered on ▲</a>
{% elif sorted_by == "discovered_on DESC" %}
<a href="/torrents/?search={{ search }}&sort_by=discovered_on+ASC">Discovered on ▼</a>
{% else %}
<a href="/torrents/?search={{ search }}&sort_by=discovered_on+DESC">Discovered on</a>
{% endif %}
</th>
<th>Name</th>
<th>Size</th>
<th>Discovered on</th>
</tr>
</thead>
<tbody>
{% for torrent in torrents %}
{{ range .Torrents }}
<tr>
<td><a href="magnet:?xt=urn:btih:{{ torrent.info_hash }}&dn={{ torrent.name }}">
<img src="static/assets/magnet.gif') }}" alt="Magnet link"
<td><a href="magnet:?xt=urn:btih:{{ bytesToHex .InfoHash }}&dn={{ .Name }}">
<img src="static/assets/magnet.gif" alt="Magnet link"
title="Download this torrent using magnet" /></a></td>
<td><a href="/torrents/{{ torrent.info_hash }}/{{ torrent.name }}">{{ torrent.name }}</a></td>
<td>{{ torrent.size }}</td>
<td>{{ torrent.discovered_on }}</td>
<td><a href="/torrents/{{ bytesToHex .InfoHash }}/{{ .Name }}">{{ .Name }}</a></td>
<td>{{ humanizeSize .Size }}</td>
<td>{{ unixTimeToYearMonthDay .DiscoveredOn }}</td>
</tr>
{% endfor %}
{{ end }}
</tbody>
</table>
</main>
<footer>
<form action="/torrents" method="get">
<button {% if page == 0 %}disabled{% endif %}>Previous</button>
<input type="text" name="search" value="{{ search }}" hidden>
{% if sorted_by %}
<input type="text" name="sort_by" value="{{ sorted_by }}" hidden>
{% endif %}
<input type="number" name="page" value="{{ page - 1 }}" hidden>
</form>
<form action="/torrents" method="get">
<button {% if not next_page_exists %}disabled{% endif %}>Next</button>
<input type="text" name="search" value="{{ search }}" hidden>
{% if sorted_by %}
<input type="text" name="sort_by" value="{{ sorted_by }}" hidden>
{% endif %}
<input type="number" name="page" value="{{ page + 1 }}" hidden>
</form>
<button onclick="loadMore();" {{ if not .CanLoadMore }} disabled {{ end }}>
Load More Results
</button>
</footer>
</body>
</html>

View File

@ -2,16 +2,20 @@ package main
import (
"encoding/hex"
"fmt"
"html/template"
"log"
"net/http"
"os"
"strconv"
"strings"
"time"
"unsafe"
//"strconv"
"strings"
// "time"
"github.com/dustin/go-humanize"
// "github.com/dustin/go-humanize"
"github.com/gorilla/mux"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
@ -26,17 +30,20 @@ var database persistence.Database
// ========= TD: TemplateData =========
type HomepageTD struct {
Count uint
NTorrents uint
}
type TorrentsTD struct {
Search string
CanLoadMore bool
Query string
SubscriptionURL string
Torrents []persistence.TorrentMetadata
Before int64
After int64
SortedBy string
NextPageExists bool
Epoch int64
LastOrderedValue uint64
LastID uint64
}
type TorrentTD struct {
@ -100,17 +107,21 @@ func main() {
"humanizeSize": func(s uint64) string {
return humanize.IBytes(s)
},
"comma": func(s uint) string {
return humanize.Comma(int64(s))
},
}
templates = make(map[string]*template.Template)
templates["feed"] = template.Must(template.New("feed").Parse(string(mustAsset("templates/feed.xml"))))
templates["homepage"] = template.Must(template.New("homepage").Parse(string(mustAsset("templates/homepage.html"))))
templates["statistics"] = template.Must(template.New("statistics").Parse(string(mustAsset("templates/statistics.html"))))
templates["torrent"] = template.Must(template.New("torrent").Funcs(templateFunctions).Parse(string(mustAsset("templates/torrent.html"))))
// templates["feed"] = template.Must(template.New("feed").Parse(string(mustAsset("templates/feed.xml"))))
templates["homepage"] = template.Must(template.New("homepage").Funcs(templateFunctions).Parse(string(mustAsset("templates/homepage.html"))))
// templates["statistics"] = template.Must(template.New("statistics").Parse(string(mustAsset("templates/statistics.html"))))
// templates["torrent"] = template.Must(template.New("torrent").Funcs(templateFunctions).Parse(string(mustAsset("templates/torrent.html"))))
templates["torrents"] = template.Must(template.New("torrents").Funcs(templateFunctions).Parse(string(mustAsset("templates/torrents.html"))))
var err error
database, err = persistence.MakeDatabase("sqlite3:///home/bora/.local/share/magneticod/database.sqlite3", unsafe.Pointer(logger))
database, err = persistence.MakeDatabase("sqlite3:///home/bora/.local/share/magneticod/database.sqlite3", logger)
if err != nil {
panic(err.Error())
}
@ -121,135 +132,85 @@ func main() {
// DONE
func rootHandler(w http.ResponseWriter, r *http.Request) {
count, err := database.GetNumberOfTorrents()
nTorrents, err := database.GetNumberOfTorrents()
if err != nil {
panic(err.Error())
}
templates["homepage"].Execute(w, HomepageTD{
Count: count,
NTorrents: nTorrents,
})
}
func respondError(w http.ResponseWriter, statusCode int, format string, a ...interface{}) {
w.WriteHeader(statusCode)
w.Write([]byte(fmt.Sprintf(format, a...)))
}
func torrentsHandler(w http.ResponseWriter, r *http.Request) {
// TODO: Parsing URL Query is tedious and looks stupid... can we do better?
queryValues := r.URL.Query()
// Parses `before` and `after` parameters in the URL query following the conditions below:
// * `before` and `after` cannot be both supplied at the same time.
// * `before` -if supplied- cannot be less than or equal to zero.
// * `after` -if supplied- cannot be greater than the current Unix time.
// * if `before` is not supplied, it is set to the current Unix time.
qBefore, qAfter := (int64)(-1), (int64)(-1)
var query string
epoch := time.Now().Unix() // epoch, if not supplied, is NOW.
var lastOrderedValue, lastID *uint64
if query = queryValues.Get("query"); query == "" {
respondError(w, 400, "query is missing")
return
}
if queryValues.Get("epoch") != "" && queryValues.Get("lastOrderedValue") != "" && queryValues.Get("lastID") != "" {
var err error
if queryValues.Get("before") != "" {
qBefore, err = strconv.ParseInt(queryValues.Get("before"), 10, 64)
epoch, err = strconv.ParseInt(queryValues.Get("epoch"), 10, 64)
if err != nil {
panic(err.Error())
respondError(w, 400, "error while parsing epoch: %s", err.Error())
return
}
if qBefore <= 0 {
panic("before parameter is less than or equal to zero!")
if epoch <= 0 {
respondError(w, 400, "epoch has to be greater than zero")
return
}
} else if queryValues.Get("after") != "" {
if qBefore != -1 {
panic("both before and after supplied")
}
qAfter, err = strconv.ParseInt(queryValues.Get("after"), 10, 64)
*lastOrderedValue, err = strconv.ParseUint(queryValues.Get("lastOrderedValue"), 10, 64)
if err != nil {
panic(err.Error())
respondError(w, 400, "error while parsing lastOrderedValue: %s", err.Error())
return
}
if qAfter > time.Now().Unix() {
panic("after parameter is greater than the current Unix time!")
}
} else {
qBefore = time.Now().Unix()
if *lastOrderedValue <= 0 {
respondError(w, 400, "lastOrderedValue has to be greater than zero")
return
}
var torrents []persistence.TorrentMetadata
if qBefore != -1 {
torrents, err = database.GetNewestTorrents(N_TORRENTS, qBefore)
} else {
torrents, err = database.QueryTorrents(
queryValues.Get("search"),
persistence.BY_DISCOVERED_ON,
true,
false,
N_TORRENTS,
qAfter,
true,
)
}
*lastID, err = strconv.ParseUint(queryValues.Get("lastID"), 10, 64)
if err != nil {
panic(err.Error())
respondError(w, 400, "error while parsing lastID: %s", err.Error())
return
}
if *lastID <= 0 {
respondError(w, 400, "lastID has to be greater than zero")
return
}
} else if !(queryValues.Get("epoch") == "" && queryValues.Get("lastOrderedValue") == "" && queryValues.Get("lastID") == "") {
respondError(w, 400, "`epoch`, `lastOrderedValue`, `lastID` must be supplied altogether, if supplied.")
return
}
// TODO: for testing, REMOVE
torrents[2].HasReadme = true
templates["torrents"].Execute(w, TorrentsTD{
Search: "",
SubscriptionURL: "borabora",
Torrents: torrents,
Before: torrents[len(torrents)-1].DiscoveredOn,
After: torrents[0].DiscoveredOn,
SortedBy: "anan",
NextPageExists: true,
})
}
func newestTorrentsHandler(w http.ResponseWriter, r *http.Request) {
queryValues := r.URL.Query()
qBefore, qAfter := (int64)(-1), (int64)(-1)
var err error
if queryValues.Get("before") != "" {
qBefore, err = strconv.ParseInt(queryValues.Get("before"), 10, 64)
torrents, err := database.QueryTorrents(query, epoch, persistence.ByRelevance, true, 20, nil, nil)
if err != nil {
panic(err.Error())
}
} else if queryValues.Get("after") != "" {
if qBefore != -1 {
panic("both before and after supplied")
}
qAfter, err = strconv.ParseInt(queryValues.Get("after"), 10, 64)
if err != nil {
panic(err.Error())
}
} else {
qBefore = time.Now().Unix()
respondError(w, 400, "query error: %s", err.Error())
return
}
var torrents []persistence.TorrentMetadata
if qBefore != -1 {
torrents, err = database.QueryTorrents(
"",
persistence.BY_DISCOVERED_ON,
true,
false,
N_TORRENTS,
qBefore,
false,
)
} else {
torrents, err = database.QueryTorrents(
"",
persistence.BY_DISCOVERED_ON,
false,
false,
N_TORRENTS,
qAfter,
true,
)
}
if err != nil {
panic(err.Error())
if torrents == nil {
panic("torrents is nil!!!")
}
templates["torrents"].Execute(w, TorrentsTD{
Search: "",
CanLoadMore: true,
Query: query,
SubscriptionURL: "borabora",
Torrents: torrents,
Before: torrents[len(torrents)-1].DiscoveredOn,
After: torrents[0].DiscoveredOn,
SortedBy: "anan",
NextPageExists: true,
})

View File

@ -27,8 +27,8 @@ type Database interface {
orderBy orderingCriteria,
ascending bool,
limit uint,
lastOrderedValue *uint,
lastID *uint,
lastOrderedValue *uint64,
lastID *uint64,
) ([]TorrentMetadata, error)
// GetTorrents returns the TorrentExtMetadata for the torrent of the given InfoHash. Will return
// nil, nil if the torrent does not exist in the database.

View File

@ -166,8 +166,8 @@ func (db *sqlite3Database) QueryTorrents(
orderBy orderingCriteria,
ascending bool,
limit uint,
lastOrderedValue *uint,
lastID *uint,
lastOrderedValue *uint64,
lastID *uint64,
) ([]TorrentMetadata, error) {
if query == "" && orderBy == ByRelevance {
return nil, fmt.Errorf("torrents cannot be ordered by relevance when the query is empty")
@ -177,7 +177,7 @@ func (db *sqlite3Database) QueryTorrents(
}
doJoin := query != ""
firstPage := lastID != nil
firstPage := true // lastID != nil
// executeTemplate is used to prepare the SQL query, WITH PLACEHOLDERS FOR USER INPUT.
sqlQuery := executeTemplate(`
@ -196,11 +196,11 @@ func (db *sqlite3Database) QueryTorrents(
) AS idx USING(id)
{{ end }}
WHERE modified_on <= ?
{{ if not FirstPage }}
{{ if not .FirstPage }}
AND id > ?
AND {{ .OrderOn }} {{ GTEorLTE(.Ascending) }} ?
AND {{ .OrderOn }} {{ GTEorLTE .Ascending }} ?
{{ end }}
ORDER BY {{ .OrderOn }} {{ AscOrDesc(.Ascending) }}, id ASC
ORDER BY {{ .OrderOn }} {{ AscOrDesc .Ascending }}, id ASC
LIMIT ?;
`, struct {
DoJoin bool
@ -208,17 +208,17 @@ func (db *sqlite3Database) QueryTorrents(
OrderOn string
Ascending bool
}{
DoJoin: doJoin, // if there is a query, do join
FirstPage: firstPage, // lastID != nil implies that lastOrderedValue != nil as well
DoJoin: doJoin,
FirstPage: firstPage,
OrderOn: orderOn(orderBy),
Ascending: ascending,
}, template.FuncMap{
"GTEorLTE": func(ascending bool) string {
// TODO: or maybe vice versa idk
if ascending {
return "<"
} else {
return ">"
} else {
return "<"
}
},
"AscOrDesc": func(ascending bool) string {
@ -236,7 +236,7 @@ func (db *sqlite3Database) QueryTorrents(
queryArgs = append(queryArgs, query)
}
queryArgs = append(queryArgs, epoch)
if firstPage {
if !firstPage {
queryArgs = append(queryArgs, lastID)
queryArgs = append(queryArgs, lastOrderedValue)
}
@ -247,8 +247,7 @@ func (db *sqlite3Database) QueryTorrents(
return nil, fmt.Errorf("error while querying torrents: %s", err.Error())
}
var torrents []TorrentMetadata
torrents := make([]TorrentMetadata, 0)
for rows.Next() {
var torrent TorrentMetadata
if err = rows.Scan(&torrent.InfoHash, &torrent.Name, &torrent.Size, &torrent.DiscoveredOn, &torrent.NFiles); err != nil {
@ -338,12 +337,14 @@ func (db *sqlite3Database) GetFiles(infoHash []byte) ([]File, error) {
}
func (db *sqlite3Database) GetStatistics(n uint, to string) (*Statistics, error) {
/*
to_time, granularity, err := ParseISO8601(to)
if err != nil {
return nil, fmt.Errorf("parsing @to error: %s", err.Error())
}
// TODO
*/
return nil, nil
}
@ -497,8 +498,8 @@ func (db *sqlite3Database) setupDatabase() error {
//
// * Added `n_files` column to the `torrents` table.
zap.L().Warn("Updating database schema from 2 to 3... (this might take a while)")
tx.Exec(`
CREATE VIRTUAL TABLE torrents_idx USING fts5(name, content='torrents', content_rowid='id', tokenize="porter unicode61 separators ' !""#$%&''()*+,-./:;<=>?@[\]^_` + "`" + `{|}~'");
_, err = tx.Exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS torrents_idx USING fts5(name, content='torrents', content_rowid='id', tokenize="porter unicode61 separators ' !""#$%&''()*+,-./:;<=>?@[\]^_` + "`" + `{|}~'");
-- Populate the index
INSERT INTO torrents_idx(rowid, name) SELECT id, name FROM torrents;
@ -517,8 +518,8 @@ func (db *sqlite3Database) setupDatabase() error {
-- Add column modified_on
ALTER TABLE torrents ADD COLUMN modified_on INTEGER;
UPDATE torrents SET modified_on = (SELECT discovered_on);
CREATE INDEX modified_on_index ON torrents (modified_on);
UPDATE torrents SET torrents.modified_on = (SELECT discovered_on);
PRAGMA user_version = 3;
`)