New qute:history page.

qutebrowser/browser/qutescheme.py

@@ -24,12 +24,12 @@ Module attributes:
     _HANDLERS: The handlers registered via decorators.
 """
 
+import json
 import sys
 import time
-import datetime
 import urllib.parse
 
-from PyQt5.QtCore import QUrlQuery
+from PyQt5.QtCore import QUrl, QUrlQuery
 
 import qutebrowser
 from qutebrowser.utils import (version, utils, jinja, log, message, docutils,
@@ -165,83 +165,77 @@ def qute_bookmarks(_url):
 
 @add_handler('history')  # noqa
 def qute_history(url):
-    """Handler for qute:history. Display history."""
+    """Handler for qute:history. Display and serve history."""
-    # Get current date from query parameter, if not given choose today.
+    def history_iter(start_time, reverse=False):
-    curr_date = datetime.date.today()
+        """Iterate through the history and get items we're interested.
-    try:
-        query_date = QUrlQuery(url).queryItemValue("date")
-        if query_date:
-            curr_date = datetime.datetime.strptime(query_date, "%Y-%m-%d")
-            curr_date = curr_date.date()
-    except ValueError:
-        log.misc.debug("Invalid date passed to qute:history: " + query_date)
 
-    one_day = datetime.timedelta(days=1)
+        Keyword arguments:
-    next_date = curr_date + one_day
+        reverse -- whether to reverse the history_dict before iterating.
-    prev_date = curr_date - one_day
+        start_time -- select history starting from this timestamp.
-
+        """
-    def history_iter(reverse):
-        """Iterate through the history and get items we're interested in."""
-        curr_timestamp = time.mktime(curr_date.timetuple())
         history = objreg.get('web-history').history_dict.values()
         if reverse:
             history = reversed(history)
 
+        end_time = start_time - 86400.0  # end is 24hrs earlier than start
+
         for item in history:
-            # If we can't apply the reverse performance trick below,
+            # Abort/continue as early as possible
-            # at least continue as early as possible with old items.
+            item_newer = item.atime > start_time
-            # This gets us down from 550ms to 123ms with 500k old items on my
+            item_older = item.atime < end_time
-            # machine.
+            if reverse:
-            if item.atime < curr_timestamp and not reverse:
+                # history_dict is reversed, we are going back in history.
-                continue
+                # so:
+                #     abort if item is older than start_time+24hr
+                #     skip if item is newer than start
+                if item_older:
+                    return
+                if item_newer:
+                    continue
+            else:
+                # history_dict is not reversed, we are going forward in history.
+                # so:
+                #     abort if item is newer than start_time
+                #     skip if item is older than start_time+24hrs
+                if item_older:
+                    continue
+                if item_newer:
+                    return
 
-            # Convert timestamp
+            # Skip items not within start_time and end_time
-            try:
-                item_atime = datetime.datetime.fromtimestamp(item.atime)
-            except (ValueError, OSError, OverflowError):
-                log.misc.debug("Invalid timestamp {}.".format(item.atime))
-                continue
-
-            if reverse and item_atime.date() < curr_date:
-                # If we could reverse the history in-place, and this entry is
-                # older than today, only older entries will follow, so we can
-                # abort here.
-                return
-
-            # Skip items not on curr_date
             # Skip redirects
             # Skip qute:// links
+            is_in_window = item.atime > end_time and item.atime <= start_time
             is_internal = item.url.scheme() == 'qute'
-            is_not_today = item_atime.date() != curr_date
+            if item.redirect or is_internal or not is_in_window:
-            if item.redirect or is_internal or is_not_today:
                 continue
 
             # Use item's url as title if there's no title.
             item_url = item.url.toDisplayString()
             item_title = item.title if item.title else item_url
-            display_atime = item_atime.strftime("%X")
+            item_time = int(item.atime)
 
-            yield (item_url, item_title, display_atime)
+            yield {"url": item_url, "title": item_title, "time": item_time}
 
-    if sys.hexversion >= 0x03050000:
+    if QUrl(url).path() == '/data':
-        # On Python >= 3.5 we can reverse the ordereddict in-place and thus
+        # Use start_time in query or current time.
-        # apply an additional performance improvement in history_iter.
+        start_time = QUrlQuery(url).queryItemValue("start_time")
-        # On my machine, this gets us down from 550ms to 72us with 500k old
+        start_time = float(start_time) if start_time else time.time()
-        # items.
+
-        history = list(history_iter(reverse=True))
+        if sys.hexversion >= 0x03050000:
+            # On Python >= 3.5 we can reverse the ordereddict in-place and thus
+            # apply an additional performance improvement in history_iter.
+            # On my machine, this gets us down from 550ms to 72us with 500k old
+            # items.
+            history = list(history_iter(start_time, reverse=True))
+        else:
+            # On Python 3.4, we can't do that, so we'd need to copy the entire
+            # history to a list. There, filter first and then reverse it here.
+            history = reversed(list(history_iter(start_time, reverse=False)))
+
+        return 'text/html', json.dumps(history)
     else:
-        # On Python 3.4, we can't do that, so we'd need to copy the entire
+        return 'text/html', jinja.render('history.html', title='History')
-        # history to a list. There, filter first and then reverse it here.
-        history = reversed(list(history_iter(reverse=False)))
-
-    html = jinja.render('history.html',
-                        title='History',
-                        history=history,
-                        curr_date=curr_date,
-                        next_date=next_date,
-                        prev_date=prev_date,
-                        today=datetime.date.today())
-    return 'text/html', html
 
 
 @add_handler('pyeval')

qutebrowser/html/history.html

@@ -16,43 +16,165 @@ td.time {
     white-space: nowrap;
 }
 
+table {
+    margin-bottom: 30px;
+}
+
 .date {
-    color: #888;
+    color: #555;
-    font-size: 14pt;
+    font-size: 12pt;
-    padding-left: 25px;
+    padding-bottom: 15px;
-}
-
-.pagination-link {
-    display: inline-block;
-    margin-bottom: 10px;
-    margin-top: 10px;
-    padding-right: 10px;
-}
-
-.pagination-link > a {
-    color: #333;
     font-weight: bold;
+    text-align: left;
+}
+
+.session-separator {
+    color: #aaa;
+    height: 40px;
+    text-align: center;
+}
+
+{% endblock %}
+
+{% block script %}
+/**
+ * Container for global stuff
+ */
+var global = {
+    // The last history item that was seen.
+    lastItem: null,
+    // The cutoff interval for session-separator (30 minutes)
+    SESSION_CUTOFF: 30*60
+};
+
+/**
+ * Finds or creates the session table>tbody to which item with given date
+ * should be added.
+ *
+ * @param {Date} date - the date of the item being added.
+ */
+var getSessionNode = function(date) {
+    var histContainer = document.getElementById('hist-container');
+
+    // Find/create table
+    var tableId = "hist-" + date.getDate() + date.getMonth() + date.getYear();
+    var table = document.getElementById(tableId);
+    if (table === null) {
+        table = document.createElement("table");
+        table.id = tableId;
+
+        caption = document.createElement("caption");
+        caption.className = "date";
+        var options = {weekday: 'long', year: 'numeric', month: 'long', day: 'numeric'};
+        caption.innerHTML = date.toLocaleDateString('en-US', options);
+        table.appendChild(caption);
+
+        // Add table to page
+        histContainer.appendChild(table);
+    }
+
+    // Find/create tbody
+    var tbody = table.lastChild;
+    if (tbody.tagName !== "TBODY") { // this is the caption
+        tbody = document.createElement("tbody");
+        table.appendChild(tbody);
+    }
+
+    // Create session-separator and new tbody if necessary
+    if (tbody.lastChild !== null && global.lastItem !== null) {
+        lastItemDate = new Date(parseInt(global.lastItem.time)*1000);
+        var interval = (lastItemDate.getTime() - date.getTime())/1000.00;
+        if (interval > global.SESSION_CUTOFF) {
+            // Add session-separator
+            var sessionSeparator = document.createElement('td');
+            sessionSeparator.className = "session-separator";
+            sessionSeparator.colSpan = 2;
+            sessionSeparator.innerHTML = "§"
+            table.appendChild(document.createElement('tr'));
+            table.lastChild.appendChild(sessionSeparator);
+
+            // Create new tbody
+            tbody = document.createElement("tbody");
+            table.appendChild(tbody);
+        }
+    }
+
+    return tbody;
+}
+
+/**
+ * Given a history item, create and return <tr> for it.
+ * param {string} itemUrl - The url for this item
+ * param {string} itemTitle - The title for this item
+ * param {string} itemTime - The formatted time for this item
+ */
+var makeHistoryRow = function(itemUrl, itemTitle, itemTime) {
+    var row = document.createElement('tr');
+
+    var title = document.createElement('td');
+    title.className = "title";
+    var link = document.createElement('a');
+    link.href = itemUrl;
+    link.innerHTML = itemTitle;
+    title.appendChild(link);
+
+    var time = document.createElement('td');
+    time.className = "time";
+    time.innerHTML = itemTime;
+
+    row.appendChild(title);
+    row.appendChild(time);
+
+    return row;
+}
+
+var getJSON = function(url, callback) {
+    var xhr = new XMLHttpRequest();
+    xhr.open('GET', url, true);
+    xhr.responseType = 'json';
+    xhr.onload = function() {
+      var status = xhr.status;
+      callback(status, xhr.response);
+    };
+    xhr.send();
+};
+
+/**
+ * Load new history.
+ */
+var loadHistory = function() {
+    url = "qute://history/data";
+    if (global.lastItem !== null) {
+        startTime = parseInt(global.lastItem.time) - 1;
+        url = "qute://history/data?start_time=" + startTime.toString();
+    }
+
+    getJSON(url, function(status, history) {
+        if (history !== undefined) {
+            for (item of history) {
+                atime = new Date(parseInt(item.time)*1000);
+                var session = getSessionNode(atime);
+                var row = makeHistoryRow(item.url, item.title, atime.toLocaleTimeString());
+                session.appendChild(row)
+                global.lastItem = item;
+            }
+        }
+    });
 }
 {% endblock %}
 
 {% block content %}
+<h1>Browsing history</h1>
+<div id="hist-container"></div>
+<script>
+window.onload = function() {
+    loadHistory();
 
-<h1>Browsing history <span class="date">{{curr_date.strftime("%a, %d %B %Y")}}</span></h1>
+    window.onscroll = function(ev) {
-
+        if ((window.innerHeight + window.scrollY) >= document.body.scrollHeight) {
-<table>
+            loadHistory();
-    <tbody>
+        }
-    {% for url, title, time in history %}
+    };
-        <tr>
+};
-            <td class="title"><a href="{{url}}">{{title}}</a></td>
+</script>
-            <td class="time">{{time}}</td>
-        </tr>
-    {% endfor %}
-    </tbody>
-</table>
-
-<span class="pagination-link"><a href="qute://history/?date={{prev_date.strftime("%Y-%m-%d")}}" rel="prev">Previous</a></span>
-{% if today >= next_date %}
-<span class="pagination-link"><a href="qute://history/?date={{next_date.strftime("%Y-%m-%d")}}" rel="next">Next</a></span>
-{% endif %}
-
 {% endblock %}

tests/end2end/features/history.feature

@@ -77,7 +77,7 @@ Feature: Page history
     Scenario: Listing history
         When I open data/numbers/3.txt
         And I open data/numbers/4.txt
-        And I open qute:history
+        And I open qute://history/data
         Then the page should contain the plaintext "3.txt"
         Then the page should contain the plaintext "4.txt"
 

tests/unit/browser/test_qutescheme.py

@@ -17,8 +17,8 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-import datetime
+import json
-import collections
+import time
 
 from PyQt5.QtCore import QUrl
 import pytest
@@ -27,30 +27,26 @@ from qutebrowser.browser import history, qutescheme
 from qutebrowser.utils import objreg
 
 
-Dates = collections.namedtuple('Dates', ['yesterday', 'today', 'tomorrow'])
-
-
 class TestHistoryHandler:
 
     """Test the qute://history endpoint."""
 
     @pytest.fixture
-    def dates(self):
+    def entries(self):
-        one_day = datetime.timedelta(days=1)
+        """Create fake history entries."""
-        today = datetime.datetime.today()
+        # create 12 history items spaced 6 hours apart, starting from now
-        tomorrow = today + one_day
+        entry_count = 12
-        yesterday = today - one_day
+        interval = 6 * 60 * 60
-        return Dates(yesterday, today, tomorrow)
+        self.now = time.time()
 
-    @pytest.fixture
+        items = []
-    def entries(self, dates):
+        for i in range(entry_count):
-        today = history.Entry(atime=str(dates.today.timestamp()),
+            entry_atime = int(self.now - i * interval)
-            url=QUrl('www.today.com'), title='today')
+            entry = history.Entry(atime=str(entry_atime),
-        tomorrow = history.Entry(atime=str(dates.tomorrow.timestamp()),
+                url=QUrl("www.x.com/" + str(i)), title="Page " + str(i))
-            url=QUrl('www.tomorrow.com'), title='tomorrow')
+            items.insert(0, entry)
-        yesterday = history.Entry(atime=str(dates.yesterday.timestamp()),
+
-            url=QUrl('www.yesterday.com'), title='yesterday')
+        return items
-        return Dates(yesterday, today, tomorrow)
 
     @pytest.fixture
     def fake_web_history(self, fake_save_manager, tmpdir):
@@ -62,78 +58,38 @@ class TestHistoryHandler:
 
     @pytest.fixture(autouse=True)
     def fake_history(self, fake_web_history, entries):
-        """Create fake history for three different days."""
+        """Create fake history."""
-        fake_web_history._add_entry(entries.yesterday)
+        for item in entries:
-        fake_web_history._add_entry(entries.today)
+            fake_web_history._add_entry(item)
-        fake_web_history._add_entry(entries.tomorrow)
         fake_web_history.save()
 
-    def test_history_without_query(self):
+    @pytest.mark.parametrize("start_time_offset, expected_item_count", [
-        """Ensure qute://history shows today's history without any query."""
+        (0, 4),
-        _mimetype, data = qutescheme.qute_history(QUrl("qute://history"))
+        (24*60*60, 4),
-        key = "<span class=\"date\">{}</span>".format(
+        (48*60*60, 4),
-            datetime.date.today().strftime("%a, %d %B %Y"))
+        (72*60*60, 0)
-        assert key in data
+    ])
-
+    def test_qutehistory_data(self, start_time_offset, expected_item_count):
-    def test_history_with_bad_query(self):
+        """Ensure qute://history/data returns correct items."""
-        """Ensure qute://history shows today's history with bad query."""
+        start_time = int(self.now) - start_time_offset
-        url = QUrl("qute://history?date=204-blaah")
+        url = QUrl("qute://history/data?start_time=" + str(start_time))
         _mimetype, data = qutescheme.qute_history(url)
-        key = "<span class=\"date\">{}</span>".format(
+        items = json.loads(data)
-            datetime.date.today().strftime("%a, %d %B %Y"))
-        assert key in data
 
-    def test_history_today(self):
+        assert len(items) == expected_item_count
-        """Ensure qute://history shows history for today."""
-        url = QUrl("qute://history")
-        _mimetype, data = qutescheme.qute_history(url)
-        assert "today" in data
-        assert "tomorrow" not in data
-        assert "yesterday" not in data
 
-    def test_history_yesterday(self, dates):
+        end_time = start_time - 24*60*60
-        """Ensure qute://history shows history for yesterday."""
+        for item in items:
-        url = QUrl("qute://history?date=" +
+            assert item['time'] <= start_time
-                dates.yesterday.strftime("%Y-%m-%d"))
+            assert item['time'] > end_time
-        _mimetype, data = qutescheme.qute_history(url)
-        assert "today" not in data
-        assert "tomorrow" not in data
-        assert "yesterday" in data
 
-    def test_history_tomorrow(self, dates):
+    def test_qute_history_benchmark(self, fake_web_history, benchmark):
-        """Ensure qute://history shows history for tomorrow."""
+        for t in range(100000):  # one history per second
-        url = QUrl("qute://history?date=" +
-                dates.tomorrow.strftime("%Y-%m-%d"))
-        _mimetype, data = qutescheme.qute_history(url)
-        assert "today" not in data
-        assert "tomorrow" in data
-        assert "yesterday" not in data
-
-    def test_no_next_link_to_future(self, dates):
-        """Ensure there's no next link pointing to the future."""
-        url = QUrl("qute://history")
-        _mimetype, data = qutescheme.qute_history(url)
-        assert "Next" not in data
-
-        url = QUrl("qute://history?date=" +
-                dates.tomorrow.strftime("%Y-%m-%d"))
-        _mimetype, data = qutescheme.qute_history(url)
-        assert "Next" not in data
-
-    def test_qute_history_benchmark(self, dates, entries, fake_web_history,
-                                    benchmark):
-        for i in range(100000):
             entry = history.Entry(
-                atime=str(dates.yesterday.timestamp()),
+                atime=str(self.now - t),
-                url=QUrl('www.yesterday.com/{}'.format(i)),
+                url=QUrl('www.x.com/{}'.format(t)),
-                title='yesterday')
+                title='x at {}'.format(t))
             fake_web_history._add_entry(entry)
-        fake_web_history._add_entry(entries.today)
-        fake_web_history._add_entry(entries.tomorrow)
 
-        url = QUrl("qute://history")
+        url = QUrl("qute://history/data?start_time={}".format(self.now))
         _mimetype, data = benchmark(qutescheme.qute_history, url)
-
-        assert "today" in data
-        assert "tomorrow" not in data
-        assert "yesterday" not in data