diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 018582abf..45ce71c27 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -24,14 +24,16 @@ Module attributes: _HANDLERS: The handlers registered via decorators. """ +import json +import os import sys import time -import datetime import urllib.parse from PyQt5.QtCore import QUrlQuery import qutebrowser +from qutebrowser.config import config from qutebrowser.utils import (version, utils, jinja, log, message, docutils, objreg) from qutebrowser.misc import objects @@ -165,83 +167,102 @@ def qute_bookmarks(_url): @add_handler('history') # noqa def qute_history(url): - """Handler for qute:history. Display history.""" - # Get current date from query parameter, if not given choose today. - curr_date = datetime.date.today() - try: - query_date = QUrlQuery(url).queryItemValue("date") - if query_date: - curr_date = datetime.datetime.strptime(query_date, "%Y-%m-%d") - curr_date = curr_date.date() - except ValueError: - log.misc.debug("Invalid date passed to qute:history: " + query_date) + """Handler for qute:history. Display and serve history.""" + def history_iter(start_time, reverse=False): + """Iterate through the history and get items we're interested. - one_day = datetime.timedelta(days=1) - next_date = curr_date + one_day - prev_date = curr_date - one_day - - def history_iter(reverse): - """Iterate through the history and get items we're interested in.""" - curr_timestamp = time.mktime(curr_date.timetuple()) + Arguments: + reverse -- whether to reverse the history_dict before iterating. + start_time -- select history starting from this timestamp. + """ history = objreg.get('web-history').history_dict.values() if reverse: history = reversed(history) + end_time = start_time - 24*60*60 # end is 24hrs earlier than start + + # when history_dict is not reversed, we need to keep track of last item + # so that we can yield its atime + last_item = None + for item in history: - # If we can't apply the reverse performance trick below, - # at least continue as early as possible with old items. - # This gets us down from 550ms to 123ms with 500k old items on my - # machine. - if item.atime < curr_timestamp and not reverse: - continue - - # Convert timestamp - try: - item_atime = datetime.datetime.fromtimestamp(item.atime) - except (ValueError, OSError, OverflowError): - log.misc.debug("Invalid timestamp {}.".format(item.atime)) - continue - - if reverse and item_atime.date() < curr_date: - # If we could reverse the history in-place, and this entry is - # older than today, only older entries will follow, so we can - # abort here. - return - - # Skip items not on curr_date # Skip redirects # Skip qute:// links - is_internal = item.url.scheme() == 'qute' - is_not_today = item_atime.date() != curr_date - if item.redirect or is_internal or is_not_today: + if item.redirect or item.url.scheme() == 'qute': continue + # Skip items out of time window + item_newer = item.atime > start_time + item_older = item.atime <= end_time + if reverse: + # history_dict is reversed, we are going back in history. + # so: + # abort if item is older than start_time+24hr + # skip if item is newer than start + if item_older: + yield {"next": int(item.atime)} + return + if item_newer: + continue + else: + # history_dict isn't reversed, we are going forward in history. + # so: + # abort if item is newer than start_time + # skip if item is older than start_time+24hrs + if item_older: + last_item = item + continue + if item_newer: + yield {"next": int(last_item.atime if last_item else -1)} + return + # Use item's url as title if there's no title. item_url = item.url.toDisplayString() item_title = item.title if item.title else item_url - display_atime = item_atime.strftime("%X") + item_time = int(item.atime * 1000) - yield (item_url, item_title, display_atime) + yield {"url": item_url, "title": item_title, "time": item_time} - if sys.hexversion >= 0x03050000: - # On Python >= 3.5 we can reverse the ordereddict in-place and thus - # apply an additional performance improvement in history_iter. - # On my machine, this gets us down from 550ms to 72us with 500k old - # items. - history = list(history_iter(reverse=True)) + # if we reached here, we had reached the end of history + yield {"next": int(last_item.atime if last_item else -1)} + + if url.path() == '/data': + # Use start_time in query or current time. + try: + start_time = QUrlQuery(url).queryItemValue("start_time") + start_time = float(start_time) if start_time else time.time() + except ValueError as e: + raise QuteSchemeError("Query parameter start_time is invalid", e) + + if sys.hexversion >= 0x03050000: + # On Python >= 3.5 we can reverse the ordereddict in-place and thus + # apply an additional performance improvement in history_iter. + # On my machine, this gets us down from 550ms to 72us with 500k old + # items. + history = history_iter(start_time, reverse=True) + else: + # On Python 3.4, we can't do that, so we'd need to copy the entire + # history to a list. There, filter first and then reverse it here. + history = reversed(list(history_iter(start_time, reverse=False))) + + return 'text/html', json.dumps(list(history)) else: - # On Python 3.4, we can't do that, so we'd need to copy the entire - # history to a list. There, filter first and then reverse it here. - history = reversed(list(history_iter(reverse=False))) + return 'text/html', jinja.render('history.html', title='History', + session_interval=config.get('ui', 'history-session-interval')) - html = jinja.render('history.html', - title='History', - history=history, - curr_date=curr_date, - next_date=next_date, - prev_date=prev_date, - today=datetime.date.today()) - return 'text/html', html + +@add_handler('javascript') +def qute_javascript(url): + """Handler for qute:javascript. + + Return content of file given as query parameter. + """ + path = url.path() + if path: + path = "javascript" + os.sep.join(path.split('/')) + return 'text/html', utils.read_file(path, binary=False) + else: + raise QuteSchemeError("No file specified", ValueError()) @add_handler('pyeval') diff --git a/qutebrowser/config/configdata.py b/qutebrowser/config/configdata.py index dab540934..25520157d 100644 --- a/qutebrowser/config/configdata.py +++ b/qutebrowser/config/configdata.py @@ -292,6 +292,12 @@ def data(readonly=False): )), ('ui', sect.KeyValue( + ('history-session-interval', + SettingValue(typ.Int(), '30'), + "The maximum time in minutes between two history items for them " + "to be considered being from the same session. Use -1 to " + "disable separation."), + ('zoom-levels', SettingValue(typ.List(typ.Perc(minval=0)), '25%,33%,50%,67%,75%,90%,100%,110%,125%,150%,175%,' diff --git a/qutebrowser/html/history.html b/qutebrowser/html/history.html index 94f82182f..58e467135 100644 --- a/qutebrowser/html/history.html +++ b/qutebrowser/html/history.html @@ -16,43 +16,75 @@ td.time { white-space: nowrap; } +table { + margin-bottom: 30px; +} + .date { - color: #888; - font-size: 14pt; - padding-left: 25px; -} - -.pagination-link { - display: inline-block; - margin-bottom: 10px; - margin-top: 10px; - padding-right: 10px; -} - -.pagination-link > a { - color: #333; + color: #555; + font-size: 12pt; + padding-bottom: 15px; font-weight: bold; + text-align: left; } -{% endblock %} +#load { + color: #555; + font-weight: bold; + text-decoration: none; +} + +#eof { + color: #aaa; + margin-bottom: 30px; + text-align: center; + width: 100%; +} + +.session-separator { + color: #aaa; + height: 40px; + text-align: center; +} + +.error { + background-color: #ffbbbb; + border-radius: 5px; + font-weight: bold; + padding: 10px; + text-align: center; + width: 100%; + border: 1px solid #ff7777; +} + +{% endblock %} {% block content %} +

Browsing history

+ +
+ + + + {% endblock %} diff --git a/qutebrowser/javascript/history.js b/qutebrowser/javascript/history.js new file mode 100644 index 000000000..a2448a14f --- /dev/null +++ b/qutebrowser/javascript/history.js @@ -0,0 +1,190 @@ +/** + * Copyright 2017 Imran Sobir + * + * This file is part of qutebrowser. + * + * qutebrowser is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * qutebrowser is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with qutebrowser. If not, see . + */ + +"use strict"; + +window.loadHistory = (function() { + // Date of last seen item. + var lastItemDate = null; + + // The time to load next. + var nextTime = null; + + // The URL to fetch data from. + var DATA_URL = "qute://history/data"; + + // Various fixed elements + var EOF_MESSAGE = document.getElementById("eof"); + var LOAD_LINK = document.getElementById("load"); + var HIST_CONTAINER = document.getElementById("hist-container"); + + /** + * Finds or creates the session table>tbody to which item with given date + * should be added. + * + * @param {Date} date - the date of the item being added. + * @returns {Element} the element to which new rows should be added. + */ + function getSessionNode(date) { + // Find/create table + var tableId = ["hist", date.getDate(), date.getMonth(), + date.getYear()].join("-"); + var table = document.getElementById(tableId); + if (table === null) { + table = document.createElement("table"); + table.id = tableId; + + // Caption contains human-readable date + var caption = document.createElement("caption"); + caption.className = "date"; + var options = { + "weekday": "long", + "year": "numeric", + "month": "long", + "day": "numeric", + }; + caption.innerHTML = date.toLocaleDateString("en-US", options); + table.appendChild(caption); + + // Add table to page + HIST_CONTAINER.appendChild(table); + } + + // Find/create tbody + var tbody = table.lastChild; + if (tbody.tagName !== "TBODY") { + tbody = document.createElement("tbody"); + table.appendChild(tbody); + } + + // Create session-separator and new tbody if necessary + if (tbody.lastChild !== null && lastItemDate !== null && + window.SESSION_INTERVAL > 0) { + var interval = lastItemDate.getTime() - date.getTime(); + if (interval > window.SESSION_INTERVAL) { + // Add session-separator + var sessionSeparator = document.createElement("td"); + sessionSeparator.className = "session-separator"; + sessionSeparator.colSpan = 2; + sessionSeparator.innerHTML = "§"; + table.appendChild(document.createElement("tr")); + table.lastChild.appendChild(sessionSeparator); + + // Create new tbody + tbody = document.createElement("tbody"); + table.appendChild(tbody); + } + } + + return tbody; + } + + /** + * Given a history item, create and return for it. + * + * @param {string} itemUrl - The url for this item. + * @param {string} itemTitle - The title for this item. + * @param {string} itemTime - The formatted time for this item. + * @returns {Element} the completed tr. + */ + function makeHistoryRow(itemUrl, itemTitle, itemTime) { + var row = document.createElement("tr"); + + var title = document.createElement("td"); + title.className = "title"; + var link = document.createElement("a"); + link.href = itemUrl; + link.innerHTML = itemTitle; + title.appendChild(link); + + var time = document.createElement("td"); + time.className = "time"; + time.innerHTML = itemTime; + + row.appendChild(title); + row.appendChild(time); + + return row; + } + + /** + * Get JSON from given URL. + * + * @param {string} url - the url to fetch data from. + * @param {function} callback - the function to callback with data. + * @returns {void} + */ + function getJSON(url, callback) { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.responseType = "json"; + xhr.onload = function() { + var status = xhr.status; + callback(status, xhr.response); + }; + xhr.send(); + } + + /** + * Receive history data from qute://history/data. + * + * @param {Number} status - The status of the query. + * @param {Array} history - History data. + * @returns {void} + */ + function receiveHistory(status, history) { + if (history === null) { + return; + } + + for (var i = 0, len = history.length - 1; i < len; i++) { + var item = history[i]; + var currentItemDate = new Date(item.time); + getSessionNode(currentItemDate).appendChild(makeHistoryRow( + item.url, item.title, currentItemDate.toLocaleTimeString() + )); + lastItemDate = currentItemDate; + } + + var next = history[history.length - 1].next; + if (next === -1) { + // Reached end of history + window.onscroll = null; + EOF_MESSAGE.style.display = "block"; + LOAD_LINK.style.display = "none"; + } else { + nextTime = next; + } + } + + /** + * Load new history. + * @return {void} + */ + function loadHistory() { + if (nextTime === null) { + getJSON(DATA_URL, receiveHistory); + } else { + var url = DATA_URL.concat("?start_time=", nextTime.toString()); + getJSON(url, receiveHistory); + } + } + + return loadHistory; +})(); diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index bc7f537e6..2df4ada21 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -77,7 +77,7 @@ Feature: Page history Scenario: Listing history When I open data/numbers/3.txt And I open data/numbers/4.txt - And I open qute:history + And I open qute://history/data Then the page should contain the plaintext "3.txt" Then the page should contain the plaintext "4.txt" diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 5ceecb7f8..e46038c8d 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -17,8 +17,9 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -import datetime -import collections +import json +import os +import time from PyQt5.QtCore import QUrl import pytest @@ -27,30 +28,72 @@ from qutebrowser.browser import history, qutescheme from qutebrowser.utils import objreg -Dates = collections.namedtuple('Dates', ['yesterday', 'today', 'tomorrow']) +class TestJavascriptHandler: + + """Test the qute://javascript endpoint.""" + + # Tuples of fake JS files and their content. + js_files = [ + ('foo.js', "var a = 'foo';"), + ('bar.js', "var a = 'bar';"), + ] + + @pytest.fixture(autouse=True) + def patch_read_file(self, monkeypatch): + """Patch utils.read_file to return few fake JS files.""" + def _read_file(path, binary=False): + """Faked utils.read_file.""" + assert not binary + for filename, content in self.js_files: + if path == os.path.join('javascript', filename): + return content + raise OSError("File not found {}!".format(path)) + + monkeypatch.setattr('qutebrowser.utils.utils.read_file', _read_file) + + @pytest.mark.parametrize("filename, content", js_files) + def test_qutejavascript(self, filename, content): + url = QUrl("qute://javascript/{}".format(filename)) + _mimetype, data = qutescheme.qute_javascript(url) + + assert data == content + + def test_qutejavascript_404(self): + url = QUrl("qute://javascript/404.js") + + with pytest.raises(qutescheme.QuteSchemeOSError): + qutescheme.data_for_url(url) + + def test_qutejavascript_empty_query(self): + url = QUrl("qute://javascript") + + with pytest.raises(qutescheme.QuteSchemeError): + qutescheme.qute_javascript(url) class TestHistoryHandler: """Test the qute://history endpoint.""" - @pytest.fixture - def dates(self): - one_day = datetime.timedelta(days=1) - today = datetime.datetime.today() - tomorrow = today + one_day - yesterday = today - one_day - return Dates(yesterday, today, tomorrow) + @pytest.fixture(scope="module") + def now(self): + return int(time.time()) @pytest.fixture - def entries(self, dates): - today = history.Entry(atime=str(dates.today.timestamp()), - url=QUrl('www.today.com'), title='today') - tomorrow = history.Entry(atime=str(dates.tomorrow.timestamp()), - url=QUrl('www.tomorrow.com'), title='tomorrow') - yesterday = history.Entry(atime=str(dates.yesterday.timestamp()), - url=QUrl('www.yesterday.com'), title='yesterday') - return Dates(yesterday, today, tomorrow) + def entries(self, now): + """Create fake history entries.""" + # create 12 history items spaced 6 hours apart, starting from now + entry_count = 12 + interval = 6 * 60 * 60 + + items = [] + for i in range(entry_count): + entry_atime = now - i * interval + entry = history.Entry(atime=str(entry_atime), + url=QUrl("www.x.com/" + str(i)), title="Page " + str(i)) + items.insert(0, entry) + + return items @pytest.fixture def fake_web_history(self, fake_save_manager, tmpdir): @@ -62,78 +105,61 @@ class TestHistoryHandler: @pytest.fixture(autouse=True) def fake_history(self, fake_web_history, entries): - """Create fake history for three different days.""" - fake_web_history._add_entry(entries.yesterday) - fake_web_history._add_entry(entries.today) - fake_web_history._add_entry(entries.tomorrow) + """Create fake history.""" + for item in entries: + fake_web_history._add_entry(item) fake_web_history.save() - def test_history_without_query(self): - """Ensure qute://history shows today's history without any query.""" - _mimetype, data = qutescheme.qute_history(QUrl("qute://history")) - key = "{}".format( - datetime.date.today().strftime("%a, %d %B %Y")) - assert key in data - - def test_history_with_bad_query(self): - """Ensure qute://history shows today's history with bad query.""" - url = QUrl("qute://history?date=204-blaah") + @pytest.mark.parametrize("start_time_offset, expected_item_count", [ + (0, 4), + (24*60*60, 4), + (48*60*60, 4), + (72*60*60, 0) + ]) + def test_qutehistory_data(self, start_time_offset, expected_item_count, + now): + """Ensure qute://history/data returns correct items.""" + start_time = now - start_time_offset + url = QUrl("qute://history/data?start_time=" + str(start_time)) _mimetype, data = qutescheme.qute_history(url) - key = "{}".format( - datetime.date.today().strftime("%a, %d %B %Y")) - assert key in data + items = json.loads(data) + items = [item for item in items if 'time' in item] # skip 'next' item - def test_history_today(self): - """Ensure qute://history shows history for today.""" - url = QUrl("qute://history") + assert len(items) == expected_item_count + + # test times + end_time = start_time - 24*60*60 + for item in items: + assert item['time'] <= start_time * 1000 + assert item['time'] > end_time * 1000 + + @pytest.mark.parametrize("start_time_offset, next_time", [ + (0, 24*60*60), + (24*60*60, 48*60*60), + (48*60*60, -1), + (72*60*60, -1) + ]) + def test_qutehistory_next(self, start_time_offset, next_time, now): + """Ensure qute://history/data returns correct items.""" + start_time = now - start_time_offset + url = QUrl("qute://history/data?start_time=" + str(start_time)) _mimetype, data = qutescheme.qute_history(url) - assert "today" in data - assert "tomorrow" not in data - assert "yesterday" not in data + items = json.loads(data) + items = [item for item in items if 'next' in item] # 'next' items + assert len(items) == 1 - def test_history_yesterday(self, dates): - """Ensure qute://history shows history for yesterday.""" - url = QUrl("qute://history?date=" + - dates.yesterday.strftime("%Y-%m-%d")) - _mimetype, data = qutescheme.qute_history(url) - assert "today" not in data - assert "tomorrow" not in data - assert "yesterday" in data + if next_time == -1: + assert items[0]["next"] == -1 + else: + assert items[0]["next"] == now - next_time - def test_history_tomorrow(self, dates): - """Ensure qute://history shows history for tomorrow.""" - url = QUrl("qute://history?date=" + - dates.tomorrow.strftime("%Y-%m-%d")) - _mimetype, data = qutescheme.qute_history(url) - assert "today" not in data - assert "tomorrow" in data - assert "yesterday" not in data - - def test_no_next_link_to_future(self, dates): - """Ensure there's no next link pointing to the future.""" - url = QUrl("qute://history") - _mimetype, data = qutescheme.qute_history(url) - assert "Next" not in data - - url = QUrl("qute://history?date=" + - dates.tomorrow.strftime("%Y-%m-%d")) - _mimetype, data = qutescheme.qute_history(url) - assert "Next" not in data - - def test_qute_history_benchmark(self, dates, entries, fake_web_history, - benchmark): - for i in range(100000): + def test_qute_history_benchmark(self, fake_web_history, benchmark, now): + for t in range(100000): # one history per second entry = history.Entry( - atime=str(dates.yesterday.timestamp()), - url=QUrl('www.yesterday.com/{}'.format(i)), - title='yesterday') + atime=str(now - t), + url=QUrl('www.x.com/{}'.format(t)), + title='x at {}'.format(t)) fake_web_history._add_entry(entry) - fake_web_history._add_entry(entries.today) - fake_web_history._add_entry(entries.tomorrow) - url = QUrl("qute://history") - _mimetype, data = benchmark(qutescheme.qute_history, url) - - assert "today" in data - assert "tomorrow" not in data - assert "yesterday" not in data + url = QUrl("qute://history/data?start_time={}".format(now)) + _mimetype, _data = benchmark(qutescheme.qute_history, url)