rnhmjoj/magnetico
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

fixed some go vet warnings, fixed formatting

Browse Source
This commit is contained in:
Bora Alper 2018-08-03 15:40:04 +03:00
parent e4bb7b5b35
commit 4b9b354171
16 changed files with 173 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Makefile
View File

@ -7,3 +7,16 @@ magneticow:
# TODO: minify files! # TODO: minify files!
go-bindata -o="magneticow/bindata.go" -prefix="magneticow/data/" magneticow/data/... go-bindata -o="magneticow/bindata.go" -prefix="magneticow/data/" magneticow/data/...
go install magnetico/magneticow go install magnetico/magneticow
test:
go test github.com/boramalper/magnetico/cmd/magneticod/...
@echo
go test github.com/boramalper/magnetico/cmd/magneticow/...
@echo
go test github.com/boramalper/magnetico/pkg/persistence/...
format:
gofmt -w cmd/magneticod
gofmt -w cmd/magneticow
gofmt -w pkg/persistence

55
cmd/magneticod/bittorrent/metadata/leech.go
View File

@ -37,12 +37,12 @@ type extDict struct {
} }
type Leech struct { type Leech struct {
infoHash [20]byte infoHash [20]byte
peerAddr *net.TCPAddr peerAddr *net.TCPAddr
ev LeechEventHandlers ev LeechEventHandlers
conn *net.TCPConn conn *net.TCPConn
clientID [20]byte clientID [20]byte
ut_metadata uint8 ut_metadata uint8
metadataReceived, metadataSize uint metadataReceived, metadataSize uint
@ -50,8 +50,8 @@ type Leech struct {
} }
type LeechEventHandlers struct { type LeechEventHandlers struct {
OnSuccess func(Metadata) // must be supplied. args: metadata OnSuccess func(Metadata) // must be supplied. args: metadata
OnError func([20]byte, error) // must be supplied. args: infohash, error OnError func([20]byte, error) // must be supplied. args: infohash, error
} }
func NewLeech(infoHash [20]byte, peerAddr *net.TCPAddr, ev LeechEventHandlers) *Leech { func NewLeech(infoHash [20]byte, peerAddr *net.TCPAddr, ev LeechEventHandlers) *Leech {
@ -86,7 +86,9 @@ func (l *Leech) doBtHandshake() error {
)) ))
// ASSERTION // ASSERTION
if len(lHandshake) != 68 { panic(fmt.Sprintf("len(lHandshake) == %d", len(lHandshake))) } if len(lHandshake) != 68 {
panic(fmt.Sprintf("len(lHandshake) == %d", len(lHandshake)))
}
err := l.writeAll(lHandshake) err := l.writeAll(lHandshake)
if err != nil { if err != nil {
@ -148,15 +150,15 @@ func (l *Leech) doExHandshake() error {
} }
func (l *Leech) requestAllPieces() error { func (l *Leech) requestAllPieces() error {
/* // reqq
* reqq // An integer, the number of outstanding request messages this client supports without
* An integer, the number of outstanding request messages this client supports without // dropping any. The default in in libtorrent is 250.
* dropping any. The default in in libtorrent is 250. //
* "handshake message" @ "Extension Protocol" @ http://www.bittorrent.org/beps/bep_0010.html // "handshake message" @ "Extension Protocol" @ http://www.bittorrent.org/beps/bep_0010.html
* //
* TODO: maybe by requesting all pieces at once we are exceeding this limit? maybe we should // TODO: maybe by requesting all pieces at once we are exceeding this limit? maybe we should
* request as we receive pieces? // request as we receive pieces?
*/
// Request all the pieces of metadata // Request all the pieces of metadata
nPieces := int(math.Ceil(float64(l.metadataSize) / math.Pow(2, 14))) nPieces := int(math.Ceil(float64(l.metadataSize) / math.Pow(2, 14)))
for piece := 0; piece < nPieces; piece++ { for piece := 0; piece < nPieces; piece++ {
@ -166,13 +168,13 @@ func (l *Leech) requestAllPieces() error {
MsgType: 0, MsgType: 0,
Piece: piece, Piece: piece,
}) })
if err != nil { // ASSERT if err != nil { // ASSERT
panic(errors.Wrap(err, "marshal extDict")) panic(errors.Wrap(err, "marshal extDict"))
} }
err = l.writeAll([]byte(fmt.Sprintf( err = l.writeAll([]byte(fmt.Sprintf(
"%s\x14%s%s", "%s\x14%s%s",
toBigEndian(uint(2 + len(extDictDump)), 4), toBigEndian(uint(2+len(extDictDump)), 4),
toBigEndian(uint(l.ut_metadata), 1), toBigEndian(uint(l.ut_metadata), 1),
extDictDump, extDictDump,
))) )))
@ -213,7 +215,7 @@ func (l *Leech) readExMessage() ([]byte, error) {
// Every extension message has at least 2 bytes. // Every extension message has at least 2 bytes.
if len(rMessage) < 2 { if len(rMessage) < 2 {
continue; continue
} }
// We are interested only in extension messages, whose first byte is always 20 // We are interested only in extension messages, whose first byte is always 20
@ -343,7 +345,7 @@ func (l *Leech) Do(deadline time.Time) {
l.OnError(fmt.Errorf("remote peer rejected sending metadata")) l.OnError(fmt.Errorf("remote peer rejected sending metadata"))
return return
} }
if rExtDict.MsgType == 1 { // data if rExtDict.MsgType == 1 { // data
// Get the unread bytes! // Get the unread bytes!
metadataPiece := rMessageBuf.Bytes() metadataPiece := rMessageBuf.Bytes()
@ -358,12 +360,12 @@ func (l *Leech) Do(deadline time.Time) {
l.OnError(fmt.Errorf("metadataPiece > 16kiB")) l.OnError(fmt.Errorf("metadataPiece > 16kiB"))
return return
} }
piece := rExtDict.Piece piece := rExtDict.Piece
// metadata[piece * 2**14: piece * 2**14 + len(metadataPiece)] = metadataPiece is how it'd be done in Python // metadata[piece * 2**14: piece * 2**14 + len(metadataPiece)] = metadataPiece is how it'd be done in Python
copy(l.metadata[piece*int(math.Pow(2, 14)):piece*int(math.Pow(2, 14))+len(metadataPiece)], metadataPiece) copy(l.metadata[piece*int(math.Pow(2, 14)):piece*int(math.Pow(2, 14))+len(metadataPiece)], metadataPiece)
l.metadataReceived += uint(len(metadataPiece)) l.metadataReceived += uint(len(metadataPiece))
// ... if the length of @metadataPiece is less than 16kiB AND metadata is NOT // ... if the length of @metadataPiece is less than 16kiB AND metadata is NOT
// complete then we err. // complete then we err.
if len(metadataPiece) < 16*1024 && l.metadataReceived != l.metadataSize { if len(metadataPiece) < 16*1024 && l.metadataReceived != l.metadataSize {
@ -377,14 +379,14 @@ func (l *Leech) Do(deadline time.Time) {
} }
} }
} }
// Verify the checksum // Verify the checksum
sha1Sum := sha1.Sum(l.metadata) sha1Sum := sha1.Sum(l.metadata)
if !bytes.Equal(sha1Sum[:], l.infoHash[:]) { if !bytes.Equal(sha1Sum[:], l.infoHash[:]) {
l.OnError(fmt.Errorf("infohash mismatch")) l.OnError(fmt.Errorf("infohash mismatch"))
return return
} }
// Check the info dictionary // Check the info dictionary
info := new(metainfo.Info) info := new(metainfo.Info)
err = bencode.Unmarshal(l.metadata, info) err = bencode.Unmarshal(l.metadata, info)
@ -420,7 +422,7 @@ func (l *Leech) Do(deadline time.Time) {
l.OnError(fmt.Errorf("file size less than zero")) l.OnError(fmt.Errorf("file size less than zero"))
return return
} }
totalSize += uint64(file.Size) totalSize += uint64(file.Size)
} }
@ -483,4 +485,3 @@ func toBigEndian(i uint, n int) []byte {
return b return b
} }

14
cmd/magneticod/bittorrent/metadata/leech_test.go
View File

@ -7,23 +7,23 @@ import (
"github.com/anacrolix/torrent/bencode" "github.com/anacrolix/torrent/bencode"
) )
var operationsTest_instances = []struct{ var operationsTest_instances = []struct {
dump []byte dump []byte
surplus []byte surplus []byte
}{ }{
// No Surplus // No Surplus
{ {
dump: []byte("d1:md11:ut_metadatai1ee13:metadata_sizei22528ee"), dump: []byte("d1:md11:ut_metadatai1ee13:metadata_sizei22528ee"),
surplus: []byte(""), surplus: []byte(""),
}, },
// Surplus is an ASCII string // Surplus is an ASCII string
{ {
dump: []byte("d1:md11:ut_metadatai1ee13:metadata_sizei22528eeDENEME"), dump: []byte("d1:md11:ut_metadatai1ee13:metadata_sizei22528eeDENEME"),
surplus: []byte("DENEME"), surplus: []byte("DENEME"),
}, },
// Surplus is a bencoded dictionary // Surplus is a bencoded dictionary
{ {
dump: []byte("d1:md11:ut_metadatai1ee13:metadata_sizei22528eed3:inti1337ee"), dump: []byte("d1:md11:ut_metadatai1ee13:metadata_sizei22528eed3:inti1337ee"),
surplus: []byte("d3:inti1337ee"), surplus: []byte("d3:inti1337ee"),
}, },
} }
@ -31,14 +31,14 @@ var operationsTest_instances = []struct{
func TestDecoder(t *testing.T) { func TestDecoder(t *testing.T) {
for i, instance := range operationsTest_instances { for i, instance := range operationsTest_instances {
buf := bytes.NewBuffer(instance.dump) buf := bytes.NewBuffer(instance.dump)
err := bencode.NewDecoder(buf).Decode(&struct {}{}) err := bencode.NewDecoder(buf).Decode(&struct{}{})
if err != nil { if err != nil {
t.Errorf("Couldn't decode the dump #%d! %s", i + 1, err.Error()) t.Errorf("Couldn't decode the dump #%d! %s", i+1, err.Error())
} }
bufSurplus := buf.Bytes() bufSurplus := buf.Bytes()
if !bytes.Equal(bufSurplus, instance.surplus) { if !bytes.Equal(bufSurplus, instance.surplus) {
t.Errorf("Surplus #%d is not equal to what we expected! `%s`", i + 1, bufSurplus) t.Errorf("Surplus #%d is not equal to what we expected! `%s`", i+1, bufSurplus)
} }
} }
} }

2
cmd/magneticod/bittorrent/metadata/sink.go
View File

@ -64,7 +64,7 @@ func (ms *Sink) Sink(res mainline.TrawlingResult) {
leech := NewLeech(res.InfoHash, res.PeerAddr, LeechEventHandlers{ leech := NewLeech(res.InfoHash, res.PeerAddr, LeechEventHandlers{
OnSuccess: ms.flush, OnSuccess: ms.flush,
OnError: ms.onLeechError, OnError: ms.onLeechError,
}) })
go leech.Do(time.Now().Add(ms.deadline)) go leech.Do(time.Now().Add(ms.deadline))

1
cmd/magneticod/dht/mainline/codec.go
View File

@ -258,7 +258,6 @@ func (e *Error) UnmarshalBencode(b []byte) (err error) {
} }
if len(matches[2]) != msgLen { if len(matches[2]) != msgLen {
return
return fmt.Errorf("error message have different lengths (%d vs %d) \"%s\"!", len(matches[2]), msgLen, matches[2]) return fmt.Errorf("error message have different lengths (%d vs %d) \"%s\"!", len(matches[2]), msgLen, matches[2])
} }

3
cmd/magneticod/dht/mainline/protocol.go
View File

@ -221,9 +221,8 @@ func (p *Protocol) CalculateToken(address net.IP) []byte {
func (p *Protocol) VerifyToken(address net.IP, token []byte) bool { func (p *Protocol) VerifyToken(address net.IP, token []byte) bool {
p.tokenLock.Lock() p.tokenLock.Lock()
defer p.tokenLock.Unlock() defer p.tokenLock.Unlock()
// TODO: implement VerifyToken()
panic("VerifyToken() not implemented yet!") panic("VerifyToken() not implemented yet!")
// TODO
return false
} }
func (p *Protocol) updateTokenSecret() { func (p *Protocol) updateTokenSecret() {

14
cmd/magneticod/dht/mainline/service.go
View File

@ -113,7 +113,7 @@ func (s *TrawlingService) bootstrap() {
if err != nil { if err != nil {
zap.L().Error("Could NOT resolve (UDP) address of the bootstrapping node!", zap.L().Error("Could NOT resolve (UDP) address of the bootstrapping node!",
zap.String("node", node)) zap.String("node", node))
continue; continue
} }
s.protocol.SendMessage(NewFindNodeQuery(s.trueNodeID, target), addr) s.protocol.SendMessage(NewFindNodeQuery(s.trueNodeID, target), addr)
@ -208,11 +208,11 @@ func (s *TrawlingService) onCongestion() {
* *
* In case of congestion, decrease the maximum number of nodes to the 90% of the current value. * In case of congestion, decrease the maximum number of nodes to the 90% of the current value.
*/ */
if s.maxNeighbors < 200 { if s.maxNeighbors < 200 {
zap.L().Warn("Max. number of neighbours are < 200 and there is still congestion!" + zap.L().Warn("Max. number of neighbours are < 200 and there is still congestion!" +
"(check your network connection if this message recurs)") "(check your network connection if this message recurs)")
return return
} }
s.maxNeighbors = uint(float32(s.maxNeighbors) * 0.9) s.maxNeighbors = uint(float32(s.maxNeighbors) * 0.9)
} }

8
cmd/magneticod/dht/mainline/transport.go
View File

@ -36,8 +36,8 @@ func NewTransport(laddr string, onMessage func(*Message, *net.UDPAddr), onConges
* *
* https://en.wikipedia.org/wiki/User_Datagram_Protocol * https://en.wikipedia.org/wiki/User_Datagram_Protocol
*/ */
t.buffer = make([]byte, 65507) t.buffer = make([]byte, 65507)
t.onMessage = onMessage t.onMessage = onMessage
t.onCongestion = onCongestion t.onCongestion = onCongestion
var err error var err error
@ -83,7 +83,7 @@ func (t *Transport) Start() {
} }
func (t *Transport) Terminate() { func (t *Transport) Terminate() {
unix.Close(t.fd); unix.Close(t.fd)
} }
// readMessages is a goroutine! // readMessages is a goroutine!
@ -131,7 +131,7 @@ func (t *Transport) WriteMessages(msg *Message, addr *net.UDPAddr) {
if addrSA == nil { if addrSA == nil {
zap.L().Debug("Wrong net address for the remote peer!", zap.L().Debug("Wrong net address for the remote peer!",
zap.String("addr", addr.String())) zap.String("addr", addr.String()))
return; return
} }
err = unix.Sendto(t.fd, data, 0, addrSA) err = unix.Sendto(t.fd, data, 0, addrSA)

10
cmd/magneticod/main.go
View File

@ -37,7 +37,7 @@ type opFlags struct {
TrawlerMlInterval time.Duration TrawlerMlInterval time.Duration
Verbosity int Verbosity int
Profile string Profile string
} }
func main() { func main() {
@ -76,7 +76,7 @@ func main() {
switch opFlags.Profile { switch opFlags.Profile {
case "cpu": case "cpu":
file, err := os.OpenFile("magneticod_cpu.prof", os.O_CREATE | os.O_WRONLY, 0755) file, err := os.OpenFile("magneticod_cpu.prof", os.O_CREATE|os.O_WRONLY, 0755)
if err != nil { if err != nil {
zap.L().Panic("Could not open the cpu profile file!", zap.Error(err)) zap.L().Panic("Could not open the cpu profile file!", zap.Error(err))
} }
@ -145,9 +145,9 @@ func parseFlags() (*opFlags, error) {
if cmdF.DatabaseURL == "" { if cmdF.DatabaseURL == "" {
opF.DatabaseURL = opF.DatabaseURL =
"sqlite3://" + "sqlite3://" +
appdirs.UserDataDir("magneticod", "", "", false) + appdirs.UserDataDir("magneticod", "", "", false) +
"/database.sqlite3" + "/database.sqlite3" +
"?_journal_mode=WAL" // https://github.com/mattn/go-sqlite3#connection-string "?_journal_mode=WAL" // https://github.com/mattn/go-sqlite3#connection-string
} else { } else {
opF.DatabaseURL = cmdF.DatabaseURL opF.DatabaseURL = cmdF.DatabaseURL
} }

9
cmd/magneticow/api.go
View File

@ -16,9 +16,8 @@ import (
func apiTorrentsHandler(w http.ResponseWriter, r *http.Request) { func apiTorrentsHandler(w http.ResponseWriter, r *http.Request) {
// @lastOrderedValue AND @lastID are either both supplied or neither of them should be supplied // @lastOrderedValue AND @lastID are either both supplied or neither of them should be supplied
// at all; and if that is NOT the case, then return an error. // at all; and if that is NOT the case, then return an error.
if q := r.URL.Query(); !( if q := r.URL.Query(); !((q.Get("lastOrderedValue") != "" && q.Get("lastID") != "") ||
(q.Get("lastOrderedValue") != "" && q.Get("lastID") != "") || (q.Get("lastOrderedValue") == "" && q.Get("lastID") == "")) {
(q.Get("lastOrderedValue") == "" && q.Get("lastID") == "")) {
respondError(w, 400, "`lastOrderedValue`, `lastID` must be supplied altogether, if supplied.") respondError(w, 400, "`lastOrderedValue`, `lastID` must be supplied altogether, if supplied.")
return return
} }
@ -43,7 +42,7 @@ func apiTorrentsHandler(w http.ResponseWriter, r *http.Request) {
if tq.Epoch == nil { if tq.Epoch == nil {
tq.Epoch = new(int64) tq.Epoch = new(int64)
*tq.Epoch = time.Now().Unix() // epoch, if not supplied, is NOW. *tq.Epoch = time.Now().Unix() // epoch, if not supplied, is NOW.
} else if *tq.Epoch <= 0 { } else if *tq.Epoch <= 0 {
respondError(w, 400, "epoch must be greater than 0") respondError(w, 400, "epoch must be greater than 0")
return return
@ -213,4 +212,4 @@ func parseOrderBy(s string) (persistence.OrderingCriteria, error) {
default: default:
return persistence.ByDiscoveredOn, fmt.Errorf("unknown orderBy string: %s", s) return persistence.ByDiscoveredOn, fmt.Errorf("unknown orderBy string: %s", s)
} }
} }

3
cmd/magneticow/handlers.go
View File

@ -10,7 +10,6 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
) )
// DONE // DONE
func rootHandler(w http.ResponseWriter, r *http.Request) { func rootHandler(w http.ResponseWriter, r *http.Request) {
nTorrents, err := database.GetNumberOfTorrents() nTorrents, err := database.GetNumberOfTorrents()
@ -128,7 +127,7 @@ func feedHandler(w http.ResponseWriter, r *http.Request) {
Title string Title string
Torrents []persistence.TorrentMetadata Torrents []persistence.TorrentMetadata
}{ }{
Title: title, Title: title,
Torrents: torrents, Torrents: torrents,
}) })
if err != nil { if err != nil {

33
cmd/magneticow/main.go
View File

@ -38,10 +38,10 @@ var decoder = schema.NewDecoder()
var templates map[string]*template.Template var templates map[string]*template.Template
var database persistence.Database var database persistence.Database
var opts struct{ var opts struct {
Addr string Addr string
Database string Database string
Credentials map[string][]byte // TODO: encapsulate credentials and mutex for safety Credentials map[string][]byte // TODO: encapsulate credentials and mutex for safety
CredentialsRWMutex sync.RWMutex CredentialsRWMutex sync.RWMutex
CredentialsPath string CredentialsPath string
} }
@ -77,9 +77,9 @@ func main() {
continue continue
} }
opts.Credentials = make(map[string][]byte) // Clear opts.Credentials opts.Credentials = make(map[string][]byte) // Clear opts.Credentials
opts.CredentialsRWMutex.Unlock() opts.CredentialsRWMutex.Unlock()
if err := loadCred(opts.CredentialsPath); err != nil { // Reload credentials if err := loadCred(opts.CredentialsPath); err != nil { // Reload credentials
zap.L().Warn("couldn't load credentials", zap.Error(err)) zap.L().Warn("couldn't load credentials", zap.Error(err))
} }
} }
@ -175,7 +175,6 @@ func respondError(w http.ResponseWriter, statusCode int, format string, a ...int
w.Write([]byte(fmt.Sprintf(format, a...))) w.Write([]byte(fmt.Sprintf(format, a...)))
} }
func mustAsset(name string) []byte { func mustAsset(name string) []byte {
data, err := Asset(name) data, err := Asset(name)
if err != nil { if err != nil {
@ -186,10 +185,10 @@ func mustAsset(name string) []byte {
func parseFlags() error { func parseFlags() error {
var cmdFlags struct { var cmdFlags struct {
Addr string `short:"a" long:"addr" description:"Address (host:port) to serve on" default:":8080"` Addr string `short:"a" long:"addr" description:"Address (host:port) to serve on" default:":8080"`
Database string `short:"d" long:"database" description:"URL of the (magneticod) database"` Database string `short:"d" long:"database" description:"URL of the (magneticod) database"`
Cred string `short:"c" long:"credentials" description:"Path to the credentials file"` Cred string `short:"c" long:"credentials" description:"Path to the credentials file"`
NoAuth bool ` long:"no-auth" description:"Disables authorisation"` NoAuth bool ` long:"no-auth" description:"Disables authorisation"`
} }
if _, err := flags.Parse(&cmdFlags); err != nil { if _, err := flags.Parse(&cmdFlags); err != nil {
@ -205,9 +204,9 @@ func parseFlags() error {
if cmdFlags.Database == "" { if cmdFlags.Database == "" {
opts.Database = opts.Database =
"sqlite3://" + "sqlite3://" +
appdirs.UserDataDir("magneticod", "", "", false) + appdirs.UserDataDir("magneticod", "", "", false) +
"/database.sqlite3" + "/database.sqlite3" +
"?_journal_mode=WAL" // https://github.com/mattn/go-sqlite3#connection-string "?_journal_mode=WAL" // https://github.com/mattn/go-sqlite3#connection-string
} else { } else {
opts.Database = cmdFlags.Database opts.Database = cmdFlags.Database
} }
@ -249,12 +248,12 @@ func loadCred(cred string) error {
line, err := reader.ReadBytes('\n') line, err := reader.ReadBytes('\n')
if err != nil { if err != nil {
if err == io.EOF { if err == io.EOF {
break; break
} }
return fmt.Errorf("error while reading line %d: %s", lineno, err.Error()) return fmt.Errorf("error while reading line %d: %s", lineno, err.Error())
} }
line = line[:len(line) - 1] // strip '\n' line = line[:len(line)-1] // strip '\n'
/* The following regex checks if the line satisfies the following conditions: /* The following regex checks if the line satisfies the following conditions:
* *
@ -292,7 +291,7 @@ func loadCred(cred string) error {
func BasicAuth(handler http.HandlerFunc, realm string) http.HandlerFunc { func BasicAuth(handler http.HandlerFunc, realm string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
username, password, ok := r.BasicAuth() username, password, ok := r.BasicAuth()
if !ok { // No credentials provided if !ok { // No credentials provided
authenticate(w, realm) authenticate(w, realm)
return return
} }
@ -300,12 +299,12 @@ func BasicAuth(handler http.HandlerFunc, realm string) http.HandlerFunc {
opts.CredentialsRWMutex.RLock() opts.CredentialsRWMutex.RLock()
hashedPassword, ok := opts.Credentials[username] hashedPassword, ok := opts.Credentials[username]
opts.CredentialsRWMutex.RUnlock() opts.CredentialsRWMutex.RUnlock()
if !ok { // User not found if !ok { // User not found
authenticate(w, realm) authenticate(w, realm)
return return
} }
if err := bcrypt.CompareHashAndPassword(hashedPassword, []byte(password)); err != nil { // Wrong password if err := bcrypt.CompareHashAndPassword(hashedPassword, []byte(password)); err != nil { // Wrong password
authenticate(w, realm) authenticate(w, realm)
return return
} }

50
cmd/magneticow/main_test.go
View File

@ -16,7 +16,7 @@ type schemaStruct struct {
} }
type schemaRStruct struct { type schemaRStruct struct {
Uint64 uint64 `schema:"ruint64,required"` // https://github.com/gorilla/schema/pull/68 Uint64 uint64 `schema:"ruint64,required"` // https://github.com/gorilla/schema/pull/68
} }
// TestSchemaUnsuppliedNil tests that unsupplied values yield nil. // TestSchemaUnsuppliedNil tests that unsupplied values yield nil.
@ -26,9 +26,15 @@ func TestSchemaUnsuppliedNil(t *testing.T) {
t.Error("decoding error", err.Error()) t.Error("decoding error", err.Error())
} }
if ss.PString != nil { t.Error("PString is not nil") } if ss.PString != nil {
if ss.PUint64 != nil { t.Error("PUint64 is not nil") } t.Error("PString is not nil")
if ss.PBool != nil { t.Error("PBool is not nil") } }
if ss.PUint64 != nil {
t.Error("PUint64 is not nil")
}
if ss.PBool != nil {
t.Error("PBool is not nil")
}
} }
// TestSchemaInvalidUint64 tests that an invalid uint64 value yields nil. // TestSchemaInvalidUint64 tests that an invalid uint64 value yields nil.
@ -38,7 +44,9 @@ func TestSchemaInvalidUint64(t *testing.T) {
ss := new(schemaStruct) ss := new(schemaStruct)
err := decoder.Decode(ss, dict) err := decoder.Decode(ss, dict)
if err == nil { t.Error("err is nil") } if err == nil {
t.Error("err is nil")
}
} }
// TestSchemaInvalidBool tests that an invalid bool value yields nil. // TestSchemaInvalidBool tests that an invalid bool value yields nil.
@ -48,18 +56,22 @@ func TestSchemaInvalidBool(t *testing.T) {
ss := new(schemaStruct) ss := new(schemaStruct)
err := decoder.Decode(ss, dict) err := decoder.Decode(ss, dict)
if err == nil { t.Error("err is nil") } if err == nil {
t.Error("err is nil")
}
} }
// TestSchemaOverflow tests that integer values greater than the maximum value a field can store // TestSchemaOverflow tests that integer values greater than the maximum value a field can store
// leads to decoding errors, rather than silent overflows. // leads to decoding errors, rather than silent overflows.
func TestSchemaOverflow(t *testing.T) { func TestSchemaOverflow(t *testing.T) {
dict := make(map[string][]string) dict := make(map[string][]string)
dict["puint64"] = []string{"18446744073709551616"} // 18,446,744,073,709,551,615 + 1 dict["puint64"] = []string{"18446744073709551616"} // 18,446,744,073,709,551,615 + 1
ss := new(schemaStruct) ss := new(schemaStruct)
err := decoder.Decode(ss, dict) err := decoder.Decode(ss, dict)
if err == nil { t.Error("err is nil") } if err == nil {
t.Error("err is nil")
}
} }
// TestSchemaEmptyString tests that empty string yields nil. // TestSchemaEmptyString tests that empty string yields nil.
@ -72,7 +84,9 @@ func TestSchemaEmptyString(t *testing.T) {
t.Error("decoding error", err.Error()) t.Error("decoding error", err.Error())
} }
if ss.PString != nil { t.Error("PString is not nil") } if ss.PString != nil {
t.Error("PString is not nil")
}
} }
// TestSchemaDefault tests if unsupplied values defaults to "zero" and doesn't err // TestSchemaDefault tests if unsupplied values defaults to "zero" and doesn't err
@ -82,14 +96,22 @@ func TestSchemaDefault(t *testing.T) {
t.Error("decoding error", err.Error()) t.Error("decoding error", err.Error())
} }
if ss.String != "" { t.Error("String is not empty") } if ss.String != "" {
if ss.Uint64 != 0 { t.Error("Uint64 is not 0") } t.Error("String is not empty")
if ss.Bool != false { t.Error("Bool is not false") } }
if ss.Uint64 != 0 {
t.Error("Uint64 is not 0")
}
if ss.Bool != false {
t.Error("Bool is not false")
}
} }
func TestSchemaRequired(t *testing.T) { func TestSchemaRequired(t *testing.T) {
rs := new(schemaRStruct) rs := new(schemaRStruct)
err := decoder.Decode(rs, make(map[string][]string)) err := decoder.Decode(rs, make(map[string][]string))
if err == nil { t.Error("err is nil") } if err == nil {
t.Error("err is nil")
}
fmt.Printf(err.Error()) fmt.Printf(err.Error())
} }

31
pkg/persistence/interface.go
View File

@ -42,6 +42,7 @@ type Database interface {
} }
type OrderingCriteria uint8 type OrderingCriteria uint8
const ( const (
ByRelevance OrderingCriteria = iota ByRelevance OrderingCriteria = iota
ByTotalSize ByTotalSize
@ -51,17 +52,19 @@ const (
ByNLeechers ByNLeechers
ByUpdatedOn ByUpdatedOn
) )
// TODO: search `swtich (orderBy)` and see if all cases are covered all the time // TODO: search `swtich (orderBy)` and see if all cases are covered all the time
type databaseEngine uint8 type databaseEngine uint8
const ( const (
Sqlite3 databaseEngine = 1 Sqlite3 databaseEngine = 1
) )
type Statistics struct { type Statistics struct {
NDiscovered map[string]uint64 `json:"nDiscovered"` NDiscovered map[string]uint64 `json:"nDiscovered"`
NFiles map[string]uint64 `json:"nFiles"` NFiles map[string]uint64 `json:"nFiles"`
TotalSize map[string]uint64 `json:"totalSize"` TotalSize map[string]uint64 `json:"totalSize"`
// All these slices below have the exact length equal to the Period. // All these slices below have the exact length equal to the Period.
//NDiscovered []uint64 `json:"nDiscovered"` //NDiscovered []uint64 `json:"nDiscovered"`
@ -69,18 +72,18 @@ type Statistics struct {
} }
type File struct { type File struct {
Size int64 `json:"size"` Size int64 `json:"size"`
Path string `json:"path"` Path string `json:"path"`
} }
type TorrentMetadata struct { type TorrentMetadata struct {
ID uint64 `json:"id"` ID uint64 `json:"id"`
InfoHash []byte `json:"infoHash"` // marshalled differently InfoHash []byte `json:"infoHash"` // marshalled differently
Name string `json:"name"` Name string `json:"name"`
Size uint64 `json:"size"` Size uint64 `json:"size"`
DiscoveredOn int64 `json:"discoveredOn"` DiscoveredOn int64 `json:"discoveredOn"`
NFiles uint `json:"nFiles"` NFiles uint `json:"nFiles"`
Relevance float64 `json:"relevance"` Relevance float64 `json:"relevance"`
} }
func (tm *TorrentMetadata) MarshalJSON() ([]byte, error) { func (tm *TorrentMetadata) MarshalJSON() ([]byte, error) {
@ -122,7 +125,7 @@ func MakeDatabase(rawURL string, logger *zap.Logger) (Database, error) {
func NewStatistics() (s *Statistics) { func NewStatistics() (s *Statistics) {
s = new(Statistics) s = new(Statistics)
s.NDiscovered = make(map[string]uint64) s.NDiscovered = make(map[string]uint64)
s.NFiles = make(map[string]uint64) s.NFiles = make(map[string]uint64)
s.TotalSize = make(map[string]uint64) s.TotalSize = make(map[string]uint64)
return return
} }

31
pkg/persistence/iso8601.go
View File

@ -7,13 +7,14 @@ import (
"time" "time"
) )
var yearRE = regexp.MustCompile(`^(\d{4})$`) var yearRE = regexp.MustCompile(`^(\d{4})$`)
var monthRE = regexp.MustCompile(`^(\d{4})-(\d{2})$`) var monthRE = regexp.MustCompile(`^(\d{4})-(\d{2})$`)
var weekRE = regexp.MustCompile(`^(\d{4})-W(\d{2})$`) var weekRE = regexp.MustCompile(`^(\d{4})-W(\d{2})$`)
var dayRE = regexp.MustCompile(`^(\d{4})-(\d{2})-(\d{2})$`) var dayRE = regexp.MustCompile(`^(\d{4})-(\d{2})-(\d{2})$`)
var hourRE = regexp.MustCompile(`^(\d{4})-(\d{2})-(\d{2})T(\d{2})$`) var hourRE = regexp.MustCompile(`^(\d{4})-(\d{2})-(\d{2})T(\d{2})$`)
type Granularity int type Granularity int
const ( const (
Year Granularity = iota Year Granularity = iota
Month Month
@ -50,7 +51,7 @@ func ParseISO8601(s string) (*time.Time, Granularity, error) {
if err != nil { if err != nil {
return nil, -1, err return nil, -1, err
} }
t := time.Date(year, time.January, week * 7, 23, 59, 59, 0, time.UTC) t := time.Date(year, time.January, week*7, 23, 59, 59, 0, time.UTC)
return &t, Week, nil return &t, Week, nil
} }
@ -122,15 +123,15 @@ func daysOfMonth(month time.Month, year int) int {
} }
func isLeap(year int) bool { func isLeap(year int) bool {
if year % 4 != 0 { if year%4 != 0 {
return false return false
} else if year % 100 != 0 { } else if year%100 != 0 {
return true return true
} else if year % 400 != 0 { } else if year%400 != 0 {
return false return false
} else { } else {
return true return true
} }
} }
func atoi(s string) int { func atoi(s string) int {
@ -153,7 +154,7 @@ func parseYear(s string) (int, error) {
func parseMonth(s string) (time.Month, error) { func parseMonth(s string) (time.Month, error) {
month := atoi(s) month := atoi(s)
if month <= 0 || month >= 13 { if month <= 0 || month >= 13 {
return time.Month(-1), fmt.Errorf("month is not in range [01, 12]") return time.Month(-1), fmt.Errorf("month is not in range [01, 12]")
} }
return time.Month(month), nil return time.Month(month), nil

34
pkg/persistence/sqlite3.go
View File

@ -147,7 +147,7 @@ func (db *sqlite3Database) GetNumberOfTorrents() (uint, error) {
defer rows.Close() defer rows.Close()
if rows.Next() != true { if rows.Next() != true {
fmt.Errorf("No rows returned from `SELECT MAX(ROWID)`") return 0, fmt.Errorf("No rows returned from `SELECT MAX(ROWID)`")
} }
var n uint var n uint
@ -174,7 +174,7 @@ func (db *sqlite3Database) QueryTorrents(
return nil, fmt.Errorf("lastOrderedValue and lastID should be supplied together, if supplied") return nil, fmt.Errorf("lastOrderedValue and lastID should be supplied together, if supplied")
} }
doJoin := query != "" doJoin := query != ""
firstPage := lastID == nil firstPage := lastID == nil
// executeTemplate is used to prepare the SQL query, WITH PLACEHOLDERS FOR USER INPUT. // executeTemplate is used to prepare the SQL query, WITH PLACEHOLDERS FOR USER INPUT.
@ -206,14 +206,14 @@ func (db *sqlite3Database) QueryTorrents(
ORDER BY {{.OrderOn}} {{AscOrDesc .Ascending}}, id {{AscOrDesc .Ascending}} ORDER BY {{.OrderOn}} {{AscOrDesc .Ascending}}, id {{AscOrDesc .Ascending}}
LIMIT ?; LIMIT ?;
`, struct { `, struct {
DoJoin bool DoJoin bool
FirstPage bool FirstPage bool
OrderOn string OrderOn string
Ascending bool Ascending bool
}{ }{
DoJoin: doJoin, DoJoin: doJoin,
FirstPage: firstPage, FirstPage: firstPage,
OrderOn: orderOn(orderBy), OrderOn: orderOn(orderBy),
Ascending: ascending, Ascending: ascending,
}, template.FuncMap{ }, template.FuncMap{
"GTEorLTE": func(ascending bool) string { "GTEorLTE": func(ascending bool) string {
@ -347,24 +347,24 @@ func (db *sqlite3Database) GetStatistics(from string, n uint) (*Statistics, erro
} }
var toTime time.Time var toTime time.Time
var timef string // time format: https://www.sqlite.org/lang_datefunc.html var timef string // time format: https://www.sqlite.org/lang_datefunc.html
switch gran { switch gran {
case Year: case Year:
toTime = fromTime.AddDate(int(n), 0, 0) toTime = fromTime.AddDate(int(n), 0, 0)
timef = "%Y" timef = "%Y"
case Month: case Month:
toTime = fromTime.AddDate(0, int(n), 0) toTime = fromTime.AddDate(0, int(n), 0)
timef = "%Y-%m" timef = "%Y-%m"
case Week: case Week:
toTime = fromTime.AddDate(0, 0, int(n) * 7) toTime = fromTime.AddDate(0, 0, int(n)*7)
timef = "%Y-%W" timef = "%Y-%W"
case Day: case Day:
toTime = fromTime.AddDate(0, 0, int(n)) toTime = fromTime.AddDate(0, 0, int(n))
timef = "%Y-%m-%d" timef = "%Y-%m-%d"
case Hour: case Hour:
toTime = fromTime.Add(time.Duration(n) * time.Hour) toTime = fromTime.Add(time.Duration(n) * time.Hour)
timef = "%Y-%m-%dT%H" timef = "%Y-%m-%dT%H"
} }
// TODO: make it faster! // TODO: make it faster!
@ -378,7 +378,7 @@ func (db *sqlite3Database) GetStatistics(from string, n uint) (*Statistics, erro
AND discovered_on >= ? AND discovered_on >= ?
AND discovered_on <= ? AND discovered_on <= ?
GROUP BY dt;`, GROUP BY dt;`,
timef), timef),
fromTime.Unix(), toTime.Unix()) fromTime.Unix(), toTime.Unix())
defer rows.Close() defer rows.Close()
if err != nil { if err != nil {
@ -478,7 +478,7 @@ func (db *sqlite3Database) setupDatabase() error {
} }
switch userVersion { switch userVersion {
case 0: // FROZEN. case 0: // FROZEN.
// Upgrade from user_version 0 to 1 // Upgrade from user_version 0 to 1
// Changes: // Changes:
// * `info_hash_index` is recreated as UNIQUE. // * `info_hash_index` is recreated as UNIQUE.
@ -493,7 +493,7 @@ func (db *sqlite3Database) setupDatabase() error {
} }
fallthrough fallthrough
case 1: // FROZEN. case 1: // FROZEN.
// Upgrade from user_version 1 to 2 // Upgrade from user_version 1 to 2
// Changes: // Changes:
// * Added `n_seeders`, `n_leechers`, and `updated_on` columns to the `torrents` table, and // * Added `n_seeders`, `n_leechers`, and `updated_on` columns to the `torrents` table, and
@ -538,7 +538,7 @@ func (db *sqlite3Database) setupDatabase() error {
} }
fallthrough fallthrough
case 2: // NOT FROZEN! (subject to change or complete removal) case 2: // NOT FROZEN! (subject to change or complete removal)
// Upgrade from user_version 2 to 3 // Upgrade from user_version 2 to 3
// Changes: // Changes:
// * Created `torrents_idx` FTS5 virtual table. // * Created `torrents_idx` FTS5 virtual table.