Merge commit '724e6b29c38a55f722f17997379f1ebe190fa6db' into imransobir/newhistory

qutebrowser/browser/qutescheme.py

@@ -24,14 +24,16 @@ Module attributes:
     _HANDLERS: The handlers registered via decorators.
 """
 
+import json
+import os
 import sys
 import time
-import datetime
 import urllib.parse
 
 from PyQt5.QtCore import QUrlQuery
 
 import qutebrowser
+from qutebrowser.config import config
 from qutebrowser.utils import (version, utils, jinja, log, message, docutils,
                                objreg)
 from qutebrowser.misc import objects
@@ -165,83 +167,102 @@ def qute_bookmarks(_url):
 
 @add_handler('history')  # noqa
 def qute_history(url):
-    """Handler for qute:history. Display history."""
+    """Handler for qute:history. Display and serve history."""
-    # Get current date from query parameter, if not given choose today.
+    def history_iter(start_time, reverse=False):
-    curr_date = datetime.date.today()
+        """Iterate through the history and get items we're interested.
-    try:
-        query_date = QUrlQuery(url).queryItemValue("date")
-        if query_date:
-            curr_date = datetime.datetime.strptime(query_date, "%Y-%m-%d")
-            curr_date = curr_date.date()
-    except ValueError:
-        log.misc.debug("Invalid date passed to qute:history: " + query_date)
 
-    one_day = datetime.timedelta(days=1)
+        Arguments:
-    next_date = curr_date + one_day
+            reverse -- whether to reverse the history_dict before iterating.
-    prev_date = curr_date - one_day
+            start_time -- select history starting from this timestamp.
-
+        """
-    def history_iter(reverse):
-        """Iterate through the history and get items we're interested in."""
-        curr_timestamp = time.mktime(curr_date.timetuple())
         history = objreg.get('web-history').history_dict.values()
         if reverse:
             history = reversed(history)
 
+        end_time = start_time - 24*60*60  # end is 24hrs earlier than start
+
+        # when history_dict is not reversed, we need to keep track of last item
+        # so that we can yield its atime
+        last_item = None
+
         for item in history:
-            # If we can't apply the reverse performance trick below,
-            # at least continue as early as possible with old items.
-            # This gets us down from 550ms to 123ms with 500k old items on my
-            # machine.
-            if item.atime < curr_timestamp and not reverse:
-                continue
-
-            # Convert timestamp
-            try:
-                item_atime = datetime.datetime.fromtimestamp(item.atime)
-            except (ValueError, OSError, OverflowError):
-                log.misc.debug("Invalid timestamp {}.".format(item.atime))
-                continue
-
-            if reverse and item_atime.date() < curr_date:
-                # If we could reverse the history in-place, and this entry is
-                # older than today, only older entries will follow, so we can
-                # abort here.
-                return
-
-            # Skip items not on curr_date
             # Skip redirects
             # Skip qute:// links
-            is_internal = item.url.scheme() == 'qute'
+            if item.redirect or item.url.scheme() == 'qute':
-            is_not_today = item_atime.date() != curr_date
-            if item.redirect or is_internal or is_not_today:
                 continue
 
+            # Skip items out of time window
+            item_newer = item.atime > start_time
+            item_older = item.atime <= end_time
+            if reverse:
+                # history_dict is reversed, we are going back in history.
+                # so:
+                #     abort if item is older than start_time+24hr
+                #     skip if item is newer than start
+                if item_older:
+                    yield {"next": int(item.atime)}
+                    return
+                if item_newer:
+                    continue
+            else:
+                # history_dict isn't reversed, we are going forward in history.
+                # so:
+                #     abort if item is newer than start_time
+                #     skip if item is older than start_time+24hrs
+                if item_older:
+                    last_item = item
+                    continue
+                if item_newer:
+                    yield {"next": int(last_item.atime if last_item else -1)}
+                    return
+
             # Use item's url as title if there's no title.
             item_url = item.url.toDisplayString()
             item_title = item.title if item.title else item_url
-            display_atime = item_atime.strftime("%X")
+            item_time = int(item.atime * 1000)
 
-            yield (item_url, item_title, display_atime)
+            yield {"url": item_url, "title": item_title, "time": item_time}
 
-    if sys.hexversion >= 0x03050000:
+        # if we reached here, we had reached the end of history
-        # On Python >= 3.5 we can reverse the ordereddict in-place and thus
+        yield {"next": int(last_item.atime if last_item else -1)}
-        # apply an additional performance improvement in history_iter.
+
-        # On my machine, this gets us down from 550ms to 72us with 500k old
+    if url.path() == '/data':
-        # items.
+        # Use start_time in query or current time.
-        history = list(history_iter(reverse=True))
+        try:
+            start_time = QUrlQuery(url).queryItemValue("start_time")
+            start_time = float(start_time) if start_time else time.time()
+        except ValueError as e:
+            raise QuteSchemeError("Query parameter start_time is invalid", e)
+
+        if sys.hexversion >= 0x03050000:
+            # On Python >= 3.5 we can reverse the ordereddict in-place and thus
+            # apply an additional performance improvement in history_iter.
+            # On my machine, this gets us down from 550ms to 72us with 500k old
+            # items.
+            history = history_iter(start_time, reverse=True)
+        else:
+            # On Python 3.4, we can't do that, so we'd need to copy the entire
+            # history to a list. There, filter first and then reverse it here.
+            history = reversed(list(history_iter(start_time, reverse=False)))
+
+        return 'text/html', json.dumps(list(history))
     else:
-        # On Python 3.4, we can't do that, so we'd need to copy the entire
+        return 'text/html', jinja.render('history.html', title='History',
-        # history to a list. There, filter first and then reverse it here.
+                session_interval=config.get('ui', 'history-session-interval'))
-        history = reversed(list(history_iter(reverse=False)))
 
-    html = jinja.render('history.html',
+
-                        title='History',
+@add_handler('javascript')
-                        history=history,
+def qute_javascript(url):
-                        curr_date=curr_date,
+    """Handler for qute:javascript.
-                        next_date=next_date,
+
-                        prev_date=prev_date,
+    Return content of file given as query parameter.
-                        today=datetime.date.today())
+    """
-    return 'text/html', html
+    path = url.path()
+    if path:
+        path = "javascript" + os.sep.join(path.split('/'))
+        return 'text/html', utils.read_file(path, binary=False)
+    else:
+        raise QuteSchemeError("No file specified", ValueError())
 
 
 @add_handler('pyeval')

qutebrowser/config/configdata.py

@@ -292,6 +292,12 @@ def data(readonly=False):
         )),
 
         ('ui', sect.KeyValue(
+            ('history-session-interval',
+             SettingValue(typ.Int(), '30'),
+             "The maximum time in minutes between two history items for them "
+             "to be considered being from the same session. Use -1 to "
+             "disable separation."),
+
             ('zoom-levels',
              SettingValue(typ.List(typ.Perc(minval=0)),
                           '25%,33%,50%,67%,75%,90%,100%,110%,125%,150%,175%,'

qutebrowser/html/history.html

@@ -16,43 +16,75 @@ td.time {
     white-space: nowrap;
 }
 
+table {
+    margin-bottom: 30px;
+}
+
 .date {
-    color: #888;
+    color: #555;
-    font-size: 14pt;
+    font-size: 12pt;
-    padding-left: 25px;
+    padding-bottom: 15px;
-}
-
-.pagination-link {
-    display: inline-block;
-    margin-bottom: 10px;
-    margin-top: 10px;
-    padding-right: 10px;
-}
-
-.pagination-link > a {
-    color: #333;
     font-weight: bold;
+    text-align: left;
 }
-{% endblock %}
 
+#load {
+    color: #555;
+    font-weight: bold;
+    text-decoration: none;
+}
+
+#eof {
+    color: #aaa;
+    margin-bottom: 30px;
+    text-align: center;
+    width: 100%;
+}
+
+.session-separator {
+    color: #aaa;
+    height: 40px;
+    text-align: center;
+}
+
+.error {
+    background-color: #ffbbbb;
+    border-radius: 5px;
+    font-weight: bold;
+    padding: 10px;
+    text-align: center;
+    width: 100%;
+    border: 1px solid #ff7777;
+}
+
+{% endblock %}
 {% block content %}
+<h1>Browsing history</h1>
+<noscript>
+    <div class="error">Javascript is required to view this page.</div>
+</noscript>
+<div id="hist-container"></div>
+<span id="eof" style="display: none">end</span>
+<a href="#" id="load" style="display: none">Show more</a>
+<script type="text/javascript" src="qute://javascript/history.js"></script>
+<script type="text/javascript">
+    window.SESSION_INTERVAL = {{session_interval}} * 60 * 1000;
 
-<h1>Browsing history <span class="date">{{curr_date.strftime("%a, %d %B %Y")}}</span></h1>
+    window.onload = function() {
+        var loadLink = document.getElementById('load');
+        loadLink.style.display = null;
+        loadLink.addEventListener('click', function(ev) {
+            ev.preventDefault();
+            window.loadHistory();
+        });
 
-<table>
+        window.onscroll = function(ev) {
-    <tbody>
+            if ((window.innerHeight + window.scrollY) >= document.body.scrollHeight) {
-    {% for url, title, time in history %}
+                window.loadHistory();
-        <tr>
+            }
-            <td class="title"><a href="{{url}}">{{title}}</a></td>
+        };
-            <td class="time">{{time}}</td>
-        </tr>
-    {% endfor %}
-    </tbody>
-</table>
-
-<span class="pagination-link"><a href="qute://history/?date={{prev_date.strftime("%Y-%m-%d")}}" rel="prev">Previous</a></span>
-{% if today >= next_date %}
-<span class="pagination-link"><a href="qute://history/?date={{next_date.strftime("%Y-%m-%d")}}" rel="next">Next</a></span>
-{% endif %}
 
+        window.loadHistory();
+    };
+</script>
 {% endblock %}

qutebrowser/javascript/history.js

@@ -0,0 +1,190 @@
+/**
+ * Copyright 2017 Imran Sobir
+ *
+ * This file is part of qutebrowser.
+ *
+ * qutebrowser is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * qutebrowser is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+"use strict";
+
+window.loadHistory = (function() {
+    // Date of last seen item.
+    var lastItemDate = null;
+
+    // The time to load next.
+    var nextTime = null;
+
+    // The URL to fetch data from.
+    var DATA_URL = "qute://history/data";
+
+    // Various fixed elements
+    var EOF_MESSAGE = document.getElementById("eof");
+    var LOAD_LINK = document.getElementById("load");
+    var HIST_CONTAINER = document.getElementById("hist-container");
+
+    /**
+     * Finds or creates the session table>tbody to which item with given date
+     * should be added.
+     *
+     * @param {Date} date - the date of the item being added.
+     * @returns {Element} the element to which new rows should be added.
+     */
+    function getSessionNode(date) {
+        // Find/create table
+        var tableId = ["hist", date.getDate(), date.getMonth(),
+            date.getYear()].join("-");
+        var table = document.getElementById(tableId);
+        if (table === null) {
+            table = document.createElement("table");
+            table.id = tableId;
+
+            // Caption contains human-readable date
+            var caption = document.createElement("caption");
+            caption.className = "date";
+            var options = {
+                "weekday": "long",
+                "year": "numeric",
+                "month": "long",
+                "day": "numeric",
+            };
+            caption.innerHTML = date.toLocaleDateString("en-US", options);
+            table.appendChild(caption);
+
+            // Add table to page
+            HIST_CONTAINER.appendChild(table);
+        }
+
+        // Find/create tbody
+        var tbody = table.lastChild;
+        if (tbody.tagName !== "TBODY") {
+            tbody = document.createElement("tbody");
+            table.appendChild(tbody);
+        }
+
+        // Create session-separator and new tbody if necessary
+        if (tbody.lastChild !== null && lastItemDate !== null &&
+                window.SESSION_INTERVAL > 0) {
+            var interval = lastItemDate.getTime() - date.getTime();
+            if (interval > window.SESSION_INTERVAL) {
+                // Add session-separator
+                var sessionSeparator = document.createElement("td");
+                sessionSeparator.className = "session-separator";
+                sessionSeparator.colSpan = 2;
+                sessionSeparator.innerHTML = "&#167;";
+                table.appendChild(document.createElement("tr"));
+                table.lastChild.appendChild(sessionSeparator);
+
+                // Create new tbody
+                tbody = document.createElement("tbody");
+                table.appendChild(tbody);
+            }
+        }
+
+        return tbody;
+    }
+
+    /**
+     * Given a history item, create and return <tr> for it.
+     *
+     * @param {string} itemUrl - The url for this item.
+     * @param {string} itemTitle - The title for this item.
+     * @param {string} itemTime - The formatted time for this item.
+     * @returns {Element} the completed tr.
+     */
+    function makeHistoryRow(itemUrl, itemTitle, itemTime) {
+        var row = document.createElement("tr");
+
+        var title = document.createElement("td");
+        title.className = "title";
+        var link = document.createElement("a");
+        link.href = itemUrl;
+        link.innerHTML = itemTitle;
+        title.appendChild(link);
+
+        var time = document.createElement("td");
+        time.className = "time";
+        time.innerHTML = itemTime;
+
+        row.appendChild(title);
+        row.appendChild(time);
+
+        return row;
+    }
+
+    /**
+     * Get JSON from given URL.
+     *
+     * @param {string} url - the url to fetch data from.
+     * @param {function} callback - the function to callback with data.
+     * @returns {void}
+     */
+    function getJSON(url, callback) {
+        var xhr = new XMLHttpRequest();
+        xhr.open("GET", url, true);
+        xhr.responseType = "json";
+        xhr.onload = function() {
+            var status = xhr.status;
+            callback(status, xhr.response);
+        };
+        xhr.send();
+    }
+
+    /**
+     * Receive history data from qute://history/data.
+     *
+     * @param {Number} status - The status of the query.
+     * @param {Array} history - History data.
+     * @returns {void}
+     */
+    function receiveHistory(status, history) {
+        if (history === null) {
+            return;
+        }
+
+        for (var i = 0, len = history.length - 1; i < len; i++) {
+            var item = history[i];
+            var currentItemDate = new Date(item.time);
+            getSessionNode(currentItemDate).appendChild(makeHistoryRow(
+                item.url, item.title, currentItemDate.toLocaleTimeString()
+            ));
+            lastItemDate = currentItemDate;
+        }
+
+        var next = history[history.length - 1].next;
+        if (next === -1) {
+            // Reached end of history
+            window.onscroll = null;
+            EOF_MESSAGE.style.display = "block";
+            LOAD_LINK.style.display = "none";
+        } else {
+            nextTime = next;
+        }
+    }
+
+    /**
+     * Load new history.
+     * @return {void}
+     */
+    function loadHistory() {
+        if (nextTime === null) {
+            getJSON(DATA_URL, receiveHistory);
+        } else {
+            var url = DATA_URL.concat("?start_time=", nextTime.toString());
+            getJSON(url, receiveHistory);
+        }
+    }
+
+    return loadHistory;
+})();

tests/end2end/features/history.feature

@@ -77,7 +77,7 @@ Feature: Page history
     Scenario: Listing history
         When I open data/numbers/3.txt
         And I open data/numbers/4.txt
-        And I open qute:history
+        And I open qute://history/data
         Then the page should contain the plaintext "3.txt"
         Then the page should contain the plaintext "4.txt"
 

tests/unit/browser/test_qutescheme.py

@@ -17,8 +17,9 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-import datetime
+import json
-import collections
+import os
+import time
 
 from PyQt5.QtCore import QUrl
 import pytest
@@ -27,30 +28,72 @@ from qutebrowser.browser import history, qutescheme
 from qutebrowser.utils import objreg
 
 
-Dates = collections.namedtuple('Dates', ['yesterday', 'today', 'tomorrow'])
+class TestJavascriptHandler:
+
+    """Test the qute://javascript endpoint."""
+
+    # Tuples of fake JS files and their content.
+    js_files = [
+        ('foo.js', "var a = 'foo';"),
+        ('bar.js', "var a = 'bar';"),
+    ]
+
+    @pytest.fixture(autouse=True)
+    def patch_read_file(self, monkeypatch):
+        """Patch utils.read_file to return few fake JS files."""
+        def _read_file(path, binary=False):
+            """Faked utils.read_file."""
+            assert not binary
+            for filename, content in self.js_files:
+                if path == os.path.join('javascript', filename):
+                    return content
+            raise OSError("File not found {}!".format(path))
+
+        monkeypatch.setattr('qutebrowser.utils.utils.read_file', _read_file)
+
+    @pytest.mark.parametrize("filename, content", js_files)
+    def test_qutejavascript(self, filename, content):
+        url = QUrl("qute://javascript/{}".format(filename))
+        _mimetype, data = qutescheme.qute_javascript(url)
+
+        assert data == content
+
+    def test_qutejavascript_404(self):
+        url = QUrl("qute://javascript/404.js")
+
+        with pytest.raises(qutescheme.QuteSchemeOSError):
+            qutescheme.data_for_url(url)
+
+    def test_qutejavascript_empty_query(self):
+        url = QUrl("qute://javascript")
+
+        with pytest.raises(qutescheme.QuteSchemeError):
+            qutescheme.qute_javascript(url)
 
 
 class TestHistoryHandler:
 
     """Test the qute://history endpoint."""
 
-    @pytest.fixture
+    @pytest.fixture(scope="module")
-    def dates(self):
+    def now(self):
-        one_day = datetime.timedelta(days=1)
+        return int(time.time())
-        today = datetime.datetime.today()
-        tomorrow = today + one_day
-        yesterday = today - one_day
-        return Dates(yesterday, today, tomorrow)
 
     @pytest.fixture
-    def entries(self, dates):
+    def entries(self, now):
-        today = history.Entry(atime=str(dates.today.timestamp()),
+        """Create fake history entries."""
-            url=QUrl('www.today.com'), title='today')
+        # create 12 history items spaced 6 hours apart, starting from now
-        tomorrow = history.Entry(atime=str(dates.tomorrow.timestamp()),
+        entry_count = 12
-            url=QUrl('www.tomorrow.com'), title='tomorrow')
+        interval = 6 * 60 * 60
-        yesterday = history.Entry(atime=str(dates.yesterday.timestamp()),
+
-            url=QUrl('www.yesterday.com'), title='yesterday')
+        items = []
-        return Dates(yesterday, today, tomorrow)
+        for i in range(entry_count):
+            entry_atime = now - i * interval
+            entry = history.Entry(atime=str(entry_atime),
+                url=QUrl("www.x.com/" + str(i)), title="Page " + str(i))
+            items.insert(0, entry)
+
+        return items
 
     @pytest.fixture
     def fake_web_history(self, fake_save_manager, tmpdir):
@@ -62,78 +105,61 @@ class TestHistoryHandler:
 
     @pytest.fixture(autouse=True)
     def fake_history(self, fake_web_history, entries):
-        """Create fake history for three different days."""
+        """Create fake history."""
-        fake_web_history._add_entry(entries.yesterday)
+        for item in entries:
-        fake_web_history._add_entry(entries.today)
+            fake_web_history._add_entry(item)
-        fake_web_history._add_entry(entries.tomorrow)
         fake_web_history.save()
 
-    def test_history_without_query(self):
+    @pytest.mark.parametrize("start_time_offset, expected_item_count", [
-        """Ensure qute://history shows today's history without any query."""
+        (0, 4),
-        _mimetype, data = qutescheme.qute_history(QUrl("qute://history"))
+        (24*60*60, 4),
-        key = "<span class=\"date\">{}</span>".format(
+        (48*60*60, 4),
-            datetime.date.today().strftime("%a, %d %B %Y"))
+        (72*60*60, 0)
-        assert key in data
+    ])
-
+    def test_qutehistory_data(self, start_time_offset, expected_item_count,
-    def test_history_with_bad_query(self):
+            now):
-        """Ensure qute://history shows today's history with bad query."""
+        """Ensure qute://history/data returns correct items."""
-        url = QUrl("qute://history?date=204-blaah")
+        start_time = now - start_time_offset
+        url = QUrl("qute://history/data?start_time=" + str(start_time))
         _mimetype, data = qutescheme.qute_history(url)
-        key = "<span class=\"date\">{}</span>".format(
+        items = json.loads(data)
-            datetime.date.today().strftime("%a, %d %B %Y"))
+        items = [item for item in items if 'time' in item]  # skip 'next' item
-        assert key in data
 
-    def test_history_today(self):
+        assert len(items) == expected_item_count
-        """Ensure qute://history shows history for today."""
+
-        url = QUrl("qute://history")
+        # test times
+        end_time = start_time - 24*60*60
+        for item in items:
+            assert item['time'] <= start_time * 1000
+            assert item['time'] > end_time * 1000
+
+    @pytest.mark.parametrize("start_time_offset, next_time", [
+        (0, 24*60*60),
+        (24*60*60, 48*60*60),
+        (48*60*60, -1),
+        (72*60*60, -1)
+    ])
+    def test_qutehistory_next(self, start_time_offset, next_time, now):
+        """Ensure qute://history/data returns correct items."""
+        start_time = now - start_time_offset
+        url = QUrl("qute://history/data?start_time=" + str(start_time))
         _mimetype, data = qutescheme.qute_history(url)
-        assert "today" in data
+        items = json.loads(data)
-        assert "tomorrow" not in data
+        items = [item for item in items if 'next' in item]  # 'next' items
-        assert "yesterday" not in data
+        assert len(items) == 1
 
-    def test_history_yesterday(self, dates):
+        if next_time == -1:
-        """Ensure qute://history shows history for yesterday."""
+            assert items[0]["next"] == -1
-        url = QUrl("qute://history?date=" +
+        else:
-                dates.yesterday.strftime("%Y-%m-%d"))
+            assert items[0]["next"] == now - next_time
-        _mimetype, data = qutescheme.qute_history(url)
-        assert "today" not in data
-        assert "tomorrow" not in data
-        assert "yesterday" in data
 
-    def test_history_tomorrow(self, dates):
+    def test_qute_history_benchmark(self, fake_web_history, benchmark, now):
-        """Ensure qute://history shows history for tomorrow."""
+        for t in range(100000):  # one history per second
-        url = QUrl("qute://history?date=" +
-                dates.tomorrow.strftime("%Y-%m-%d"))
-        _mimetype, data = qutescheme.qute_history(url)
-        assert "today" not in data
-        assert "tomorrow" in data
-        assert "yesterday" not in data
-
-    def test_no_next_link_to_future(self, dates):
-        """Ensure there's no next link pointing to the future."""
-        url = QUrl("qute://history")
-        _mimetype, data = qutescheme.qute_history(url)
-        assert "Next" not in data
-
-        url = QUrl("qute://history?date=" +
-                dates.tomorrow.strftime("%Y-%m-%d"))
-        _mimetype, data = qutescheme.qute_history(url)
-        assert "Next" not in data
-
-    def test_qute_history_benchmark(self, dates, entries, fake_web_history,
-                                    benchmark):
-        for i in range(100000):
             entry = history.Entry(
-                atime=str(dates.yesterday.timestamp()),
+                atime=str(now - t),
-                url=QUrl('www.yesterday.com/{}'.format(i)),
+                url=QUrl('www.x.com/{}'.format(t)),
-                title='yesterday')
+                title='x at {}'.format(t))
             fake_web_history._add_entry(entry)
-        fake_web_history._add_entry(entries.today)
-        fake_web_history._add_entry(entries.tomorrow)
 
-        url = QUrl("qute://history")
+        url = QUrl("qute://history/data?start_time={}".format(now))
-        _mimetype, data = benchmark(qutescheme.qute_history, url)
+        _mimetype, _data = benchmark(qutescheme.qute_history, url)
-
-        assert "today" in data
-        assert "tomorrow" not in data
-        assert "yesterday" not in data