magneticow & api: statistics are now working!

cmd/magneticow/api.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/boramalper/magnetico/pkg/persistence"
@@ -94,11 +95,48 @@ func apiFilesInfohashHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func apiStatisticsHandler(w http.ResponseWriter, r *http.Request) {
+	from := r.URL.Query().Get("from")
 
+	// TODO: use gorilla?
+	var n int64
+	nStr := r.URL.Query().Get("n")
+	if nStr == "" {
+		n = 0
+	} else {
+		var err error
+		n, err = strconv.ParseInt(nStr, 10, 32)
+		if err != nil {
+			respondError(w, 400, "couldn't parse n: %s", err.Error())
+			return
+		} else if n <= 0 {
+			respondError(w, 400, "n must be a positive number")
+			return
+		}
+	}
+
+	stats, err := database.GetStatistics(from, uint(n))
+	if err != nil {
+		respondError(w, 400, "error while getting statistics: %s", err.Error())
+		return
+	}
+
+	// TODO: use plain Marshal
+	jm, err := json.MarshalIndent(stats, "", "  ")
+	if err != nil {
+		respondError(w, 500, "json marshalling error: %s", err.Error())
+		return
+	}
+
+	if _, err = w.Write(jm); err != nil {
+		zap.L().Warn("couldn't write http.ResponseWriter", zap.Error(err))
+	}
 }
 
 func parseOrderBy(s string) (persistence.OrderingCriteria, error) {
 	switch s {
+	case "RELEVANCE":
+		return persistence.ByRelevance, nil
+
 	case "TOTAL_SIZE":
 		return persistence.ByTotalSize, nil
 

cmd/magneticow/data/static/scripts/common.js

@@ -0,0 +1,61 @@
+"use strict";
+
+// Source: https://stackoverflow.com/a/111545/4466589
+function encodeQueryData(data) {
+    let ret = [];
+    for (let d in data) {
+        if (data[d] === null || data[d] === undefined)
+            continue;
+        ret.push(encodeURIComponent(d) + "=" + encodeURIComponent(data[d]));
+    }
+    return ret.join("&");
+}
+
+
+// Source: https://stackoverflow.com/q/10420352/4466589
+function fileSize(fileSizeInBytes) {
+    let i = -1;
+    let byteUnits = [' KiB', ' MiB', ' GiB', ' TiB', ' PiB', ' EiB', ' ZiB', ' YiB'];
+    do {
+        fileSizeInBytes = fileSizeInBytes / 1024;
+        i++;
+    } while (fileSizeInBytes > 1024);
+
+    return Math.max(fileSizeInBytes, 0.1).toFixed(1) + byteUnits[i];
+}
+
+/**
+ * Returns the ISO 8601 week number for this date.
+ *
+ * Source: https://stackoverflow.com/a/9047794/4466589
+ *
+ * @param int dowOffset
+ * @return int
+ */
+Date.prototype.getWeek = function (dowOffset) {
+    /* getWeek() was developed by Nick Baicoianu at MeanFreePath: http://www.meanfreepath.com */
+
+    dowOffset = 1;
+    let newYear = new Date(this.getFullYear(),0,1);
+    let day = newYear.getDay() - dowOffset; //the day of week the year begins on
+    day = (day >= 0 ? day : day + 7);
+    let daynum = Math.floor((this.getTime() - newYear.getTime() -
+        (this.getTimezoneOffset()-newYear.getTimezoneOffset())*60000)/86400000) + 1;
+    let weeknum;
+    // if the year starts before the middle of a week
+    if(day < 4) {
+        weeknum = Math.floor((daynum+day-1)/7) + 1;
+        if(weeknum > 52) {
+            nYear = new Date(this.getFullYear() + 1,0,1);
+            nday = nYear.getDay() - dowOffset;
+            nday = nday >= 0 ? nday : nday + 7;
+            /*if the next year starts before the middle of
+              the week, it is week #1 of that year*/
+            weeknum = nday < 4 ? 1 : 53;
+        }
+    }
+    else {
+        weeknum = Math.floor((daynum+day-1)/7);
+    }
+    return weeknum;
+};

cmd/magneticow/data/static/scripts/statistics.js

@@ -1,15 +1,142 @@
 "use strict";
 
+let nElem    = null,
+    unitElem = null
+;
 
 window.onload = function() {
-    Plotly.newPlot("discoveryRateGraph", discoveryRateData, {
-        title: "New Discovered Torrents Per Day in the Past 30 Days",
+    nElem    = document.getElementById("n");
+    unitElem = document.getElementById("unit");
+
+    nElem.onchange = unitElem.onchange = load;
+
+    load();
+};
+
+function plot(stats) {
+    Plotly.newPlot("nDiscovered", [{
+        x: Object.keys(stats.nDiscovered),
+        y: Object.values(stats.nDiscovered),
+        mode: "lines+markers"
+    }], {
+        title: "Torrents Discovered",
         xaxis: {
-            title: "Days",
-            tickformat: "%d %B"
+            title: "Date / Time",
         },
         yaxis: {
-            title: "Amount of Torrents Discovered"
+            title: "Number of Torrents Discovered",
         }
     });
-};
+
+    Plotly.newPlot("nFiles", [{
+        x: Object.keys(stats.nFiles),
+        y: Object.values(stats.nFiles),
+        mode: "lines+markers"
+    }], {
+        title: "Files Discovered",
+        xaxis: {
+            title: "Date / Time",
+        },
+        yaxis: {
+            title: "Number of Files Discovered",
+        }
+    });
+
+    let totalSize = Object.values(stats.totalSize);
+    for (let i in totalSize) {
+        totalSize[i] = totalSize[i] / (1024 * 1024 * 1024);
+    }
+
+    Plotly.newPlot("totalSize", [{
+        x: Object.keys(stats.totalSize),
+        y: totalSize,
+        mode: "lines+markers"
+    }], {
+        title: "Total Size of Files Discovered",
+        xaxis: {
+            title: "Date / Time",
+        },
+        yaxis: {
+            title: "Total Size of Files Discovered (in TiB)",
+        }
+    });
+}
+
+
+function load() {
+    const n = nElem.valueAsNumber;
+    const unit = unitElem.options[unitElem.selectedIndex].value;
+
+    const reqURL = "/api/v0.1/statistics?" + encodeQueryData({
+        from: fromString(n, unit),
+        n   : n,
+    });
+    console.log("reqURL", reqURL);
+
+    let req = new XMLHttpRequest();
+    req.onreadystatechange = function() {
+        if (req.readyState !== 4)
+            return;
+
+        if (req.status !== 200)
+            alert(req.responseText);
+
+        let stats = JSON.parse(req.responseText);
+        plot(stats);
+    };
+
+    req.open("GET", reqURL);
+    req.send();
+}
+
+
+function fromString(n, unit) {
+    const from = new Date(Date.now() - n * unit2seconds(unit) * 1000);
+    console.log("frommmm", unit, unit2seconds(unit), from);
+
+    let str = "" + from.getUTCFullYear();
+    if (unit === "years")
+        return str;
+    else if (unit === "weeks") {
+        str += "-W" + leftpad(from.getWeek());
+        return str;
+    } else {
+        str += "-" + leftpad(from.getUTCMonth() + 1);
+        if (unit === "months")
+            return str;
+
+        str += "-" + leftpad(from.getUTCDate());
+        if (unit === "days")
+            return str;
+
+        str += "T" + leftpad(from.getUTCHours());
+        if (unit === "hours")
+            return str;
+    }
+
+    return str;
+
+
+    function unit2seconds(u) {
+        if (u === "hours")  return            60 * 60;
+        if (u === "days")   return       24 * 60 * 60;
+        if (u === "weeks")  return   7 * 24 * 60 * 60;
+        if (u === "months") return  30 * 24 * 60 * 60;
+        if (u === "years")  return 365 * 24 * 60 * 60;
+    }
+
+
+    // pad x to minimum of n characters with c
+    function leftpad(x, n, c) {
+        if (n === undefined)
+            n = 2;
+        if (c === undefined)
+            c = "0";
+
+        const xs = "" + x;
+        if (n > xs.length)
+            return c.repeat(n - xs.length) + x;
+        else
+            return x;
+    }
+}

cmd/magneticow/data/static/scripts/torrents.js

@@ -117,28 +117,3 @@ function load() {
     req.open("GET", reqURL);
     req.send();
 }
-
-
-// Source: https://stackoverflow.com/a/111545/4466589
-function encodeQueryData(data) {
-    let ret = [];
-    for (let d in data) {
-        if (data[d] === null || data[d] === undefined)
-            continue;
-        ret.push(encodeURIComponent(d) + "=" + encodeURIComponent(data[d]));
-    }
-    return ret.join("&");
-}
-
-
-// https://stackoverflow.com/q/10420352/4466589
-function fileSize(fileSizeInBytes) {
-    let i = -1;
-    let byteUnits = [' KiB', ' MiB', ' GiB', ' TiB', ' PiB', ' EiB', ' ZiB', ' YiB'];
-    do {
-        fileSizeInBytes = fileSizeInBytes / 1024;
-        i++;
-    } while (fileSizeInBytes > 1024);
-
-    return Math.max(fileSizeInBytes, 0.1).toFixed(1) + byteUnits[i];
-}

cmd/magneticow/data/static/styles/statistics.css

@@ -19,3 +19,12 @@ header a {
     border-style: solid;
     border-width: 1px;
 }
+
+
+#options {
+    margin-bottom: 2em;
+}
+
+#options #n {
+    width: 3em;
+}

cmd/magneticow/data/templates/statistics.html

@@ -3,27 +3,37 @@
 <head>
     <meta charset="utf-8">
     <title>Statistics - magneticow</title>
+
     <link rel="stylesheet" href="static/styles/reset.css">
     <link rel="stylesheet" href="static/styles/essential.css">
     <link rel="stylesheet" href="static/styles/statistics.css">
+
     <script defer src="static/scripts/plotly-v1.26.1.min.js"></script>
+    <script defer src="static/scripts/common.js"></script>
     <script defer src="static/scripts/statistics.js"></script>
 </head>
 <body>
 <header>
-    <script>
-        var discoveryRateData = [{
-            x: {{ dates | safe }},
-            y: {{ amounts | safe }},
-            mode: "lines+markers"
-        }];
-    </script>
-
     <div><a href="/"><b>magnetico<sup>w</sup></b></a>&#8203;<sub>(pre-alpha)</sub></div>
 </header>
 
 <main>
-    <div id="discoveryRateGraph"></div>
+    <div id="options">
+        <p>Show statistics for the past...
+
+        <input id="n" title="maximum number of time units from now backwards" type="number" value="24" min="5" max="365">
+        <select id="unit" title="time unit to be used" required>
+            <!-- values are in seconds -->
+            <option value="hours" selected>Hours</option>
+            <option value="days">Days</option>
+            <option value="weeks">Weeks</option>
+            <option value="months">Months</option> <!-- 30 days -->
+            <option value="years">Years</option> <!-- 365 days -->
+        </select>.</p>
+    </div>
+    <div class="graph" id="nDiscovered"></div>
+    <div class="graph" id="nFiles"></div>
+    <div class="graph" id="totalSize"></div>
 </main>
 </body>
 </html>

cmd/magneticow/data/templates/torrents.html

@@ -9,6 +9,7 @@
     <link rel="stylesheet" href="static/styles/torrents.css">
 
     <script src="static/scripts/mustache-v2.3.0.min.js"></script>
+    <script src="static/scripts/common.js"></script>
     <script src="static/scripts/torrents.js"></script>
 
     <script id="row-template" type="text/x-handlebars-template">

cmd/magneticow/main.go

@@ -200,31 +200,11 @@ func torrentsInfohashHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// TODO: we might as well move statistics.html into static...
 func statisticsHandler(w http.ResponseWriter, r *http.Request) {
-	torrents, err := database.QueryTorrents(
-		"",
-		time.Now().Unix(),
-		persistence.ByDiscoveredOn,
-		false,
-		20,
-		nil,
-		nil,
-	)
-	if err != nil {
-		respondError(w, 400, err.Error())
-		return
-	}
-
-	err = templates["homepage"].Execute(w, struct {
-		Title    string
-		Torrents []persistence.TorrentMetadata
-	}{
-		Title: "TODO",
-		Torrents: torrents,
-	})
-	if err != nil {
-		panic(err.Error())
-	}
+	data := mustAsset("templates/statistics.html")
+	w.Header().Set("Content-Type", http.DetectContentType(data))
+	w.Write(data)
 }
 
 func feedHandler(w http.ResponseWriter, r *http.Request) {

pkg/persistence/interface.go

@@ -38,7 +38,7 @@ type Database interface {
 	// nil, nil if the torrent does not exist in the database.
 	GetTorrent(infoHash []byte) (*TorrentMetadata, error)
 	GetFiles(infoHash []byte) ([]File, error)
-	GetStatistics(n uint, to string) (*Statistics, error)
+	GetStatistics(from string, n uint) (*Statistics, error)
 }
 
 type OrderingCriteria uint8
@@ -59,12 +59,13 @@ const (
 )
 
 type Statistics struct {
-	N uint64
+	NDiscovered map[string]uint64    `json:"nDiscovered"`
+	NFiles      map[string]uint64  `json:"nFiles"`
+	TotalSize   map[string]uint64  `json:"totalSize"`
 
 	// All these slices below have the exact length equal to the Period.
-	NTorrents []uint64
-	NFiles    []uint64
-	TotalSize []uint64
+	//NDiscovered []uint64  `json:"nDiscovered"`
+
 }
 
 type File struct {
@@ -117,3 +118,11 @@ func MakeDatabase(rawURL string, logger *zap.Logger) (Database, error) {
 		return nil, fmt.Errorf("unknown URI scheme (database engine)!")
 	}
 }
+
+func NewStatistics() (s *Statistics) {
+	s = new(Statistics)
+	s.NDiscovered = make(map[string]uint64)
+	s.NFiles      = make(map[string]uint64)
+	s.TotalSize   = make(map[string]uint64)
+	return
+}

pkg/persistence/sqlite3.go

@@ -4,10 +4,10 @@ import (
 	"bytes"
 	"database/sql"
 	"fmt"
-	"text/template"
 	"net/url"
 	"os"
 	"path"
+	"text/template"
 	"time"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -351,17 +351,67 @@ func (db *sqlite3Database) GetFiles(infoHash []byte) ([]File, error) {
 	return files, nil
 }
 
-func (db *sqlite3Database) GetStatistics(n uint, to string) (*Statistics, error) {
-	/*
-	to_time, granularity, err := ParseISO8601(to)
+func (db *sqlite3Database) GetStatistics(from string, n uint) (*Statistics, error) {
+	fromTime, gran, err := ParseISO8601(from)
 	if err != nil {
-		return nil, fmt.Errorf("parsing @to error: %s", err.Error())
+		return nil, fmt.Errorf("error while parsing from: %s", err.Error())
 	}
 
-	// TODO
-	*/
+	var toTime time.Time
+	var timef  string  // time format: https://www.sqlite.org/lang_datefunc.html
 
-	return nil, nil
+	switch gran {
+	case Year:
+		toTime = fromTime.AddDate(int(n), 0, 0)
+		timef  = "%Y"
+	case Month:
+		toTime = fromTime.AddDate(0, int(n), 0)
+		timef  = "%Y-%m"
+	case Week:
+		toTime = fromTime.AddDate(0, 0, int(n) * 7)
+		timef  = "%Y-%W"
+	case Day:
+		toTime = fromTime.AddDate(0, 0, int(n))
+		timef  = "%Y-%m-%d"
+	case Hour:
+		toTime = fromTime.Add(time.Duration(n) * time.Hour)
+		timef  = "%Y-%m-%dT%H"
+	}
+
+	// TODO: make it faster!
+	rows, err := db.conn.Query(fmt.Sprintf(`
+			SELECT strftime('%s', discovered_on, 'unixepoch') AS dT
+                 , sum(files.size) AS tS
+                 , count(DISTINCT torrents.id) AS nD              
+                 , count(DISTINCT files.id) AS nF
+			FROM torrents, files
+ 			WHERE     torrents.id = files.torrent_id
+                  AND discovered_on >= ?
+                  AND discovered_on <= ?
+			GROUP BY dt;`,
+			timef),
+		fromTime.Unix(), toTime.Unix())
+	if err != nil {
+		return nil, err
+	}
+
+	stats := NewStatistics()
+
+	for rows.Next() {
+		var dT string
+		var tS, nD, nF uint64
+		if err := rows.Scan(&dT, &tS, &nD, &nF); err != nil {
+			if err := rows.Close(); err != nil {
+				panic(err.Error())
+			}
+			return nil, err
+		}
+		stats.NDiscovered[dT] = nD
+		stats.TotalSize[dT] = tS
+		stats.NFiles[dT] = nF
+	}
+
+	return stats, nil
 }
 
 func (db *sqlite3Database) setupDatabase() error {