From ba1be368cfef228e4e942fd9a96fa99d285c4b22 Mon Sep 17 00:00:00 2001 From: Bora Alper Date: Sat, 7 Jul 2018 14:56:34 +0300 Subject: [PATCH] magneticow & api: statistics are now working! --- cmd/magneticow/api.go | 38 +++++ cmd/magneticow/data/static/scripts/common.js | 61 ++++++++ .../data/static/scripts/statistics.js | 139 +++++++++++++++++- .../data/static/scripts/torrents.js | 25 ---- .../data/static/styles/statistics.css | 9 ++ cmd/magneticow/data/templates/statistics.html | 28 ++-- cmd/magneticow/data/templates/torrents.html | 1 + cmd/magneticow/main.go | 28 +--- pkg/persistence/interface.go | 19 ++- pkg/persistence/sqlite3.go | 66 ++++++++- 10 files changed, 337 insertions(+), 77 deletions(-) create mode 100644 cmd/magneticow/data/static/scripts/common.js diff --git a/cmd/magneticow/api.go b/cmd/magneticow/api.go index a317da8..8ecd8e4 100644 --- a/cmd/magneticow/api.go +++ b/cmd/magneticow/api.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "strconv" "time" "github.com/boramalper/magnetico/pkg/persistence" @@ -94,11 +95,48 @@ func apiFilesInfohashHandler(w http.ResponseWriter, r *http.Request) { } func apiStatisticsHandler(w http.ResponseWriter, r *http.Request) { + from := r.URL.Query().Get("from") + // TODO: use gorilla? + var n int64 + nStr := r.URL.Query().Get("n") + if nStr == "" { + n = 0 + } else { + var err error + n, err = strconv.ParseInt(nStr, 10, 32) + if err != nil { + respondError(w, 400, "couldn't parse n: %s", err.Error()) + return + } else if n <= 0 { + respondError(w, 400, "n must be a positive number") + return + } + } + + stats, err := database.GetStatistics(from, uint(n)) + if err != nil { + respondError(w, 400, "error while getting statistics: %s", err.Error()) + return + } + + // TODO: use plain Marshal + jm, err := json.MarshalIndent(stats, "", " ") + if err != nil { + respondError(w, 500, "json marshalling error: %s", err.Error()) + return + } + + if _, err = w.Write(jm); err != nil { + zap.L().Warn("couldn't write http.ResponseWriter", zap.Error(err)) + } } func parseOrderBy(s string) (persistence.OrderingCriteria, error) { switch s { + case "RELEVANCE": + return persistence.ByRelevance, nil + case "TOTAL_SIZE": return persistence.ByTotalSize, nil diff --git a/cmd/magneticow/data/static/scripts/common.js b/cmd/magneticow/data/static/scripts/common.js new file mode 100644 index 0000000..f935705 --- /dev/null +++ b/cmd/magneticow/data/static/scripts/common.js @@ -0,0 +1,61 @@ +"use strict"; + +// Source: https://stackoverflow.com/a/111545/4466589 +function encodeQueryData(data) { + let ret = []; + for (let d in data) { + if (data[d] === null || data[d] === undefined) + continue; + ret.push(encodeURIComponent(d) + "=" + encodeURIComponent(data[d])); + } + return ret.join("&"); +} + + +// Source: https://stackoverflow.com/q/10420352/4466589 +function fileSize(fileSizeInBytes) { + let i = -1; + let byteUnits = [' KiB', ' MiB', ' GiB', ' TiB', ' PiB', ' EiB', ' ZiB', ' YiB']; + do { + fileSizeInBytes = fileSizeInBytes / 1024; + i++; + } while (fileSizeInBytes > 1024); + + return Math.max(fileSizeInBytes, 0.1).toFixed(1) + byteUnits[i]; +} + +/** + * Returns the ISO 8601 week number for this date. + * + * Source: https://stackoverflow.com/a/9047794/4466589 + * + * @param int dowOffset + * @return int + */ +Date.prototype.getWeek = function (dowOffset) { + /* getWeek() was developed by Nick Baicoianu at MeanFreePath: http://www.meanfreepath.com */ + + dowOffset = 1; + let newYear = new Date(this.getFullYear(),0,1); + let day = newYear.getDay() - dowOffset; //the day of week the year begins on + day = (day >= 0 ? day : day + 7); + let daynum = Math.floor((this.getTime() - newYear.getTime() - + (this.getTimezoneOffset()-newYear.getTimezoneOffset())*60000)/86400000) + 1; + let weeknum; + // if the year starts before the middle of a week + if(day < 4) { + weeknum = Math.floor((daynum+day-1)/7) + 1; + if(weeknum > 52) { + nYear = new Date(this.getFullYear() + 1,0,1); + nday = nYear.getDay() - dowOffset; + nday = nday >= 0 ? nday : nday + 7; + /*if the next year starts before the middle of + the week, it is week #1 of that year*/ + weeknum = nday < 4 ? 1 : 53; + } + } + else { + weeknum = Math.floor((daynum+day-1)/7); + } + return weeknum; +}; \ No newline at end of file diff --git a/cmd/magneticow/data/static/scripts/statistics.js b/cmd/magneticow/data/static/scripts/statistics.js index 5ee83ff..2c851da 100644 --- a/cmd/magneticow/data/static/scripts/statistics.js +++ b/cmd/magneticow/data/static/scripts/statistics.js @@ -1,15 +1,142 @@ "use strict"; +let nElem = null, + unitElem = null +; window.onload = function() { - Plotly.newPlot("discoveryRateGraph", discoveryRateData, { - title: "New Discovered Torrents Per Day in the Past 30 Days", + nElem = document.getElementById("n"); + unitElem = document.getElementById("unit"); + + nElem.onchange = unitElem.onchange = load; + + load(); +}; + +function plot(stats) { + Plotly.newPlot("nDiscovered", [{ + x: Object.keys(stats.nDiscovered), + y: Object.values(stats.nDiscovered), + mode: "lines+markers" + }], { + title: "Torrents Discovered", xaxis: { - title: "Days", - tickformat: "%d %B" + title: "Date / Time", }, yaxis: { - title: "Amount of Torrents Discovered" + title: "Number of Torrents Discovered", } }); -}; + + Plotly.newPlot("nFiles", [{ + x: Object.keys(stats.nFiles), + y: Object.values(stats.nFiles), + mode: "lines+markers" + }], { + title: "Files Discovered", + xaxis: { + title: "Date / Time", + }, + yaxis: { + title: "Number of Files Discovered", + } + }); + + let totalSize = Object.values(stats.totalSize); + for (let i in totalSize) { + totalSize[i] = totalSize[i] / (1024 * 1024 * 1024); + } + + Plotly.newPlot("totalSize", [{ + x: Object.keys(stats.totalSize), + y: totalSize, + mode: "lines+markers" + }], { + title: "Total Size of Files Discovered", + xaxis: { + title: "Date / Time", + }, + yaxis: { + title: "Total Size of Files Discovered (in TiB)", + } + }); +} + + +function load() { + const n = nElem.valueAsNumber; + const unit = unitElem.options[unitElem.selectedIndex].value; + + const reqURL = "/api/v0.1/statistics?" + encodeQueryData({ + from: fromString(n, unit), + n : n, + }); + console.log("reqURL", reqURL); + + let req = new XMLHttpRequest(); + req.onreadystatechange = function() { + if (req.readyState !== 4) + return; + + if (req.status !== 200) + alert(req.responseText); + + let stats = JSON.parse(req.responseText); + plot(stats); + }; + + req.open("GET", reqURL); + req.send(); +} + + +function fromString(n, unit) { + const from = new Date(Date.now() - n * unit2seconds(unit) * 1000); + console.log("frommmm", unit, unit2seconds(unit), from); + + let str = "" + from.getUTCFullYear(); + if (unit === "years") + return str; + else if (unit === "weeks") { + str += "-W" + leftpad(from.getWeek()); + return str; + } else { + str += "-" + leftpad(from.getUTCMonth() + 1); + if (unit === "months") + return str; + + str += "-" + leftpad(from.getUTCDate()); + if (unit === "days") + return str; + + str += "T" + leftpad(from.getUTCHours()); + if (unit === "hours") + return str; + } + + return str; + + + function unit2seconds(u) { + if (u === "hours") return 60 * 60; + if (u === "days") return 24 * 60 * 60; + if (u === "weeks") return 7 * 24 * 60 * 60; + if (u === "months") return 30 * 24 * 60 * 60; + if (u === "years") return 365 * 24 * 60 * 60; + } + + + // pad x to minimum of n characters with c + function leftpad(x, n, c) { + if (n === undefined) + n = 2; + if (c === undefined) + c = "0"; + + const xs = "" + x; + if (n > xs.length) + return c.repeat(n - xs.length) + x; + else + return x; + } +} diff --git a/cmd/magneticow/data/static/scripts/torrents.js b/cmd/magneticow/data/static/scripts/torrents.js index 75fa01e..5b5a758 100644 --- a/cmd/magneticow/data/static/scripts/torrents.js +++ b/cmd/magneticow/data/static/scripts/torrents.js @@ -117,28 +117,3 @@ function load() { req.open("GET", reqURL); req.send(); } - - -// Source: https://stackoverflow.com/a/111545/4466589 -function encodeQueryData(data) { - let ret = []; - for (let d in data) { - if (data[d] === null || data[d] === undefined) - continue; - ret.push(encodeURIComponent(d) + "=" + encodeURIComponent(data[d])); - } - return ret.join("&"); -} - - -// https://stackoverflow.com/q/10420352/4466589 -function fileSize(fileSizeInBytes) { - let i = -1; - let byteUnits = [' KiB', ' MiB', ' GiB', ' TiB', ' PiB', ' EiB', ' ZiB', ' YiB']; - do { - fileSizeInBytes = fileSizeInBytes / 1024; - i++; - } while (fileSizeInBytes > 1024); - - return Math.max(fileSizeInBytes, 0.1).toFixed(1) + byteUnits[i]; -} diff --git a/cmd/magneticow/data/static/styles/statistics.css b/cmd/magneticow/data/static/styles/statistics.css index 0e0d916..d7023ea 100644 --- a/cmd/magneticow/data/static/styles/statistics.css +++ b/cmd/magneticow/data/static/styles/statistics.css @@ -18,4 +18,13 @@ header a { margin-right: auto; border-style: solid; border-width: 1px; +} + + +#options { + margin-bottom: 2em; +} + +#options #n { + width: 3em; } \ No newline at end of file diff --git a/cmd/magneticow/data/templates/statistics.html b/cmd/magneticow/data/templates/statistics.html index 9df4ab9..258f227 100644 --- a/cmd/magneticow/data/templates/statistics.html +++ b/cmd/magneticow/data/templates/statistics.html @@ -3,27 +3,37 @@ Statistics - magneticow + + +
- -
magneticow​(pre-alpha)
-
+
+

Show statistics for the past... + + + .

+
+
+
+
diff --git a/cmd/magneticow/data/templates/torrents.html b/cmd/magneticow/data/templates/torrents.html index af39683..b670362 100644 --- a/cmd/magneticow/data/templates/torrents.html +++ b/cmd/magneticow/data/templates/torrents.html @@ -9,6 +9,7 @@ +