From 845f21b275bf438eccd7854f7f5401233ec6719a Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Sun, 26 Feb 2017 17:07:30 +0500 Subject: [PATCH 01/21] New qute:history page. --- qutebrowser/browser/qutescheme.py | 116 ++++++++------- qutebrowser/html/history.html | 186 ++++++++++++++++++++----- tests/end2end/features/history.feature | 2 +- tests/unit/browser/test_qutescheme.py | 126 ++++++----------- 4 files changed, 251 insertions(+), 179 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 018582abf..e5b581fca 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -24,12 +24,12 @@ Module attributes: _HANDLERS: The handlers registered via decorators. """ +import json import sys import time -import datetime import urllib.parse -from PyQt5.QtCore import QUrlQuery +from PyQt5.QtCore import QUrl, QUrlQuery import qutebrowser from qutebrowser.utils import (version, utils, jinja, log, message, docutils, @@ -165,83 +165,77 @@ def qute_bookmarks(_url): @add_handler('history') # noqa def qute_history(url): - """Handler for qute:history. Display history.""" - # Get current date from query parameter, if not given choose today. - curr_date = datetime.date.today() - try: - query_date = QUrlQuery(url).queryItemValue("date") - if query_date: - curr_date = datetime.datetime.strptime(query_date, "%Y-%m-%d") - curr_date = curr_date.date() - except ValueError: - log.misc.debug("Invalid date passed to qute:history: " + query_date) + """Handler for qute:history. Display and serve history.""" + def history_iter(start_time, reverse=False): + """Iterate through the history and get items we're interested. - one_day = datetime.timedelta(days=1) - next_date = curr_date + one_day - prev_date = curr_date - one_day - - def history_iter(reverse): - """Iterate through the history and get items we're interested in.""" - curr_timestamp = time.mktime(curr_date.timetuple()) + Keyword arguments: + reverse -- whether to reverse the history_dict before iterating. + start_time -- select history starting from this timestamp. + """ history = objreg.get('web-history').history_dict.values() if reverse: history = reversed(history) + end_time = start_time - 86400.0 # end is 24hrs earlier than start + for item in history: - # If we can't apply the reverse performance trick below, - # at least continue as early as possible with old items. - # This gets us down from 550ms to 123ms with 500k old items on my - # machine. - if item.atime < curr_timestamp and not reverse: - continue + # Abort/continue as early as possible + item_newer = item.atime > start_time + item_older = item.atime < end_time + if reverse: + # history_dict is reversed, we are going back in history. + # so: + # abort if item is older than start_time+24hr + # skip if item is newer than start + if item_older: + return + if item_newer: + continue + else: + # history_dict is not reversed, we are going forward in history. + # so: + # abort if item is newer than start_time + # skip if item is older than start_time+24hrs + if item_older: + continue + if item_newer: + return - # Convert timestamp - try: - item_atime = datetime.datetime.fromtimestamp(item.atime) - except (ValueError, OSError, OverflowError): - log.misc.debug("Invalid timestamp {}.".format(item.atime)) - continue - - if reverse and item_atime.date() < curr_date: - # If we could reverse the history in-place, and this entry is - # older than today, only older entries will follow, so we can - # abort here. - return - - # Skip items not on curr_date + # Skip items not within start_time and end_time # Skip redirects # Skip qute:// links + is_in_window = item.atime > end_time and item.atime <= start_time is_internal = item.url.scheme() == 'qute' - is_not_today = item_atime.date() != curr_date - if item.redirect or is_internal or is_not_today: + if item.redirect or is_internal or not is_in_window: continue # Use item's url as title if there's no title. item_url = item.url.toDisplayString() item_title = item.title if item.title else item_url - display_atime = item_atime.strftime("%X") + item_time = int(item.atime) - yield (item_url, item_title, display_atime) + yield {"url": item_url, "title": item_title, "time": item_time} - if sys.hexversion >= 0x03050000: - # On Python >= 3.5 we can reverse the ordereddict in-place and thus - # apply an additional performance improvement in history_iter. - # On my machine, this gets us down from 550ms to 72us with 500k old - # items. - history = list(history_iter(reverse=True)) + if QUrl(url).path() == '/data': + # Use start_time in query or current time. + start_time = QUrlQuery(url).queryItemValue("start_time") + start_time = float(start_time) if start_time else time.time() + + if sys.hexversion >= 0x03050000: + # On Python >= 3.5 we can reverse the ordereddict in-place and thus + # apply an additional performance improvement in history_iter. + # On my machine, this gets us down from 550ms to 72us with 500k old + # items. + history = list(history_iter(start_time, reverse=True)) + else: + # On Python 3.4, we can't do that, so we'd need to copy the entire + # history to a list. There, filter first and then reverse it here. + history = reversed(list(history_iter(start_time, reverse=False))) + + return 'text/html', json.dumps(history) else: - # On Python 3.4, we can't do that, so we'd need to copy the entire - # history to a list. There, filter first and then reverse it here. - history = reversed(list(history_iter(reverse=False))) - - html = jinja.render('history.html', - title='History', - history=history, - curr_date=curr_date, - next_date=next_date, - prev_date=prev_date, - today=datetime.date.today()) - return 'text/html', html + return 'text/html', jinja.render('history.html', title='History') @add_handler('pyeval') diff --git a/qutebrowser/html/history.html b/qutebrowser/html/history.html index 94f82182f..ba64316e4 100644 --- a/qutebrowser/html/history.html +++ b/qutebrowser/html/history.html @@ -16,43 +16,165 @@ td.time { white-space: nowrap; } +table { + margin-bottom: 30px; +} + .date { - color: #888; - font-size: 14pt; - padding-left: 25px; -} - -.pagination-link { - display: inline-block; - margin-bottom: 10px; - margin-top: 10px; - padding-right: 10px; -} - -.pagination-link > a { - color: #333; + color: #555; + font-size: 12pt; + padding-bottom: 15px; font-weight: bold; + text-align: left; +} + +.session-separator { + color: #aaa; + height: 40px; + text-align: center; +} + +{% endblock %} + +{% block script %} +/** + * Container for global stuff + */ +var global = { + // The last history item that was seen. + lastItem: null, + // The cutoff interval for session-separator (30 minutes) + SESSION_CUTOFF: 30*60 +}; + +/** + * Finds or creates the session table>tbody to which item with given date + * should be added. + * + * @param {Date} date - the date of the item being added. + */ +var getSessionNode = function(date) { + var histContainer = document.getElementById('hist-container'); + + // Find/create table + var tableId = "hist-" + date.getDate() + date.getMonth() + date.getYear(); + var table = document.getElementById(tableId); + if (table === null) { + table = document.createElement("table"); + table.id = tableId; + + caption = document.createElement("caption"); + caption.className = "date"; + var options = {weekday: 'long', year: 'numeric', month: 'long', day: 'numeric'}; + caption.innerHTML = date.toLocaleDateString('en-US', options); + table.appendChild(caption); + + // Add table to page + histContainer.appendChild(table); + } + + // Find/create tbody + var tbody = table.lastChild; + if (tbody.tagName !== "TBODY") { // this is the caption + tbody = document.createElement("tbody"); + table.appendChild(tbody); + } + + // Create session-separator and new tbody if necessary + if (tbody.lastChild !== null && global.lastItem !== null) { + lastItemDate = new Date(parseInt(global.lastItem.time)*1000); + var interval = (lastItemDate.getTime() - date.getTime())/1000.00; + if (interval > global.SESSION_CUTOFF) { + // Add session-separator + var sessionSeparator = document.createElement('td'); + sessionSeparator.className = "session-separator"; + sessionSeparator.colSpan = 2; + sessionSeparator.innerHTML = "§" + table.appendChild(document.createElement('tr')); + table.lastChild.appendChild(sessionSeparator); + + // Create new tbody + tbody = document.createElement("tbody"); + table.appendChild(tbody); + } + } + + return tbody; +} + +/** + * Given a history item, create and return for it. + * param {string} itemUrl - The url for this item + * param {string} itemTitle - The title for this item + * param {string} itemTime - The formatted time for this item + */ +var makeHistoryRow = function(itemUrl, itemTitle, itemTime) { + var row = document.createElement('tr'); + + var title = document.createElement('td'); + title.className = "title"; + var link = document.createElement('a'); + link.href = itemUrl; + link.innerHTML = itemTitle; + title.appendChild(link); + + var time = document.createElement('td'); + time.className = "time"; + time.innerHTML = itemTime; + + row.appendChild(title); + row.appendChild(time); + + return row; +} + +var getJSON = function(url, callback) { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, true); + xhr.responseType = 'json'; + xhr.onload = function() { + var status = xhr.status; + callback(status, xhr.response); + }; + xhr.send(); +}; + +/** + * Load new history. + */ +var loadHistory = function() { + url = "qute://history/data"; + if (global.lastItem !== null) { + startTime = parseInt(global.lastItem.time) - 1; + url = "qute://history/data?start_time=" + startTime.toString(); + } + + getJSON(url, function(status, history) { + if (history !== undefined) { + for (item of history) { + atime = new Date(parseInt(item.time)*1000); + var session = getSessionNode(atime); + var row = makeHistoryRow(item.url, item.title, atime.toLocaleTimeString()); + session.appendChild(row) + global.lastItem = item; + } + } + }); } {% endblock %} {% block content %} +

Browsing history

+
+ {% endblock %} diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index bc7f537e6..2df4ada21 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -77,7 +77,7 @@ Feature: Page history Scenario: Listing history When I open data/numbers/3.txt And I open data/numbers/4.txt - And I open qute:history + And I open qute://history/data Then the page should contain the plaintext "3.txt" Then the page should contain the plaintext "4.txt" diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 5ceecb7f8..6cdeb13f2 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -17,8 +17,8 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -import datetime -import collections +import json +import time from PyQt5.QtCore import QUrl import pytest @@ -27,30 +27,26 @@ from qutebrowser.browser import history, qutescheme from qutebrowser.utils import objreg -Dates = collections.namedtuple('Dates', ['yesterday', 'today', 'tomorrow']) - - class TestHistoryHandler: """Test the qute://history endpoint.""" @pytest.fixture - def dates(self): - one_day = datetime.timedelta(days=1) - today = datetime.datetime.today() - tomorrow = today + one_day - yesterday = today - one_day - return Dates(yesterday, today, tomorrow) + def entries(self): + """Create fake history entries.""" + # create 12 history items spaced 6 hours apart, starting from now + entry_count = 12 + interval = 6 * 60 * 60 + self.now = time.time() - @pytest.fixture - def entries(self, dates): - today = history.Entry(atime=str(dates.today.timestamp()), - url=QUrl('www.today.com'), title='today') - tomorrow = history.Entry(atime=str(dates.tomorrow.timestamp()), - url=QUrl('www.tomorrow.com'), title='tomorrow') - yesterday = history.Entry(atime=str(dates.yesterday.timestamp()), - url=QUrl('www.yesterday.com'), title='yesterday') - return Dates(yesterday, today, tomorrow) + items = [] + for i in range(entry_count): + entry_atime = int(self.now - i * interval) + entry = history.Entry(atime=str(entry_atime), + url=QUrl("www.x.com/" + str(i)), title="Page " + str(i)) + items.insert(0, entry) + + return items @pytest.fixture def fake_web_history(self, fake_save_manager, tmpdir): @@ -62,78 +58,38 @@ class TestHistoryHandler: @pytest.fixture(autouse=True) def fake_history(self, fake_web_history, entries): - """Create fake history for three different days.""" - fake_web_history._add_entry(entries.yesterday) - fake_web_history._add_entry(entries.today) - fake_web_history._add_entry(entries.tomorrow) + """Create fake history.""" + for item in entries: + fake_web_history._add_entry(item) fake_web_history.save() - def test_history_without_query(self): - """Ensure qute://history shows today's history without any query.""" - _mimetype, data = qutescheme.qute_history(QUrl("qute://history")) - key = "{}".format( - datetime.date.today().strftime("%a, %d %B %Y")) - assert key in data - - def test_history_with_bad_query(self): - """Ensure qute://history shows today's history with bad query.""" - url = QUrl("qute://history?date=204-blaah") + @pytest.mark.parametrize("start_time_offset, expected_item_count", [ + (0, 4), + (24*60*60, 4), + (48*60*60, 4), + (72*60*60, 0) + ]) + def test_qutehistory_data(self, start_time_offset, expected_item_count): + """Ensure qute://history/data returns correct items.""" + start_time = int(self.now) - start_time_offset + url = QUrl("qute://history/data?start_time=" + str(start_time)) _mimetype, data = qutescheme.qute_history(url) - key = "{}".format( - datetime.date.today().strftime("%a, %d %B %Y")) - assert key in data + items = json.loads(data) - def test_history_today(self): - """Ensure qute://history shows history for today.""" - url = QUrl("qute://history") - _mimetype, data = qutescheme.qute_history(url) - assert "today" in data - assert "tomorrow" not in data - assert "yesterday" not in data + assert len(items) == expected_item_count - def test_history_yesterday(self, dates): - """Ensure qute://history shows history for yesterday.""" - url = QUrl("qute://history?date=" + - dates.yesterday.strftime("%Y-%m-%d")) - _mimetype, data = qutescheme.qute_history(url) - assert "today" not in data - assert "tomorrow" not in data - assert "yesterday" in data + end_time = start_time - 24*60*60 + for item in items: + assert item['time'] <= start_time + assert item['time'] > end_time - def test_history_tomorrow(self, dates): - """Ensure qute://history shows history for tomorrow.""" - url = QUrl("qute://history?date=" + - dates.tomorrow.strftime("%Y-%m-%d")) - _mimetype, data = qutescheme.qute_history(url) - assert "today" not in data - assert "tomorrow" in data - assert "yesterday" not in data - - def test_no_next_link_to_future(self, dates): - """Ensure there's no next link pointing to the future.""" - url = QUrl("qute://history") - _mimetype, data = qutescheme.qute_history(url) - assert "Next" not in data - - url = QUrl("qute://history?date=" + - dates.tomorrow.strftime("%Y-%m-%d")) - _mimetype, data = qutescheme.qute_history(url) - assert "Next" not in data - - def test_qute_history_benchmark(self, dates, entries, fake_web_history, - benchmark): - for i in range(100000): + def test_qute_history_benchmark(self, fake_web_history, benchmark): + for t in range(100000): # one history per second entry = history.Entry( - atime=str(dates.yesterday.timestamp()), - url=QUrl('www.yesterday.com/{}'.format(i)), - title='yesterday') + atime=str(self.now - t), + url=QUrl('www.x.com/{}'.format(t)), + title='x at {}'.format(t)) fake_web_history._add_entry(entry) - fake_web_history._add_entry(entries.today) - fake_web_history._add_entry(entries.tomorrow) - url = QUrl("qute://history") + url = QUrl("qute://history/data?start_time={}".format(self.now)) _mimetype, data = benchmark(qutescheme.qute_history, url) - - assert "today" in data - assert "tomorrow" not in data - assert "yesterday" not in data From 76bf8c0049cd5385c8c662d746e842a80f2a3d2f Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Sun, 26 Feb 2017 19:58:14 +0500 Subject: [PATCH 02/21] Convert history to list before converting to JSON. --- qutebrowser/browser/qutescheme.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index e5b581fca..fe22738f9 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -227,13 +227,13 @@ def qute_history(url): # apply an additional performance improvement in history_iter. # On my machine, this gets us down from 550ms to 72us with 500k old # items. - history = list(history_iter(start_time, reverse=True)) + history = history_iter(start_time, reverse=True) else: # On Python 3.4, we can't do that, so we'd need to copy the entire # history to a list. There, filter first and then reverse it here. history = reversed(list(history_iter(start_time, reverse=False))) - return 'text/html', json.dumps(history) + return 'text/html', json.dumps(list(history)) else: return 'text/html', jinja.render('history.html', title='History') From c223f6c69daf419257ec11dad306194a4901bb94 Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Sun, 26 Feb 2017 20:13:05 +0500 Subject: [PATCH 03/21] Style/misc fixes. --- qutebrowser/browser/qutescheme.py | 17 +++++++---------- tests/unit/browser/test_qutescheme.py | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index fe22738f9..6308e20fe 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -169,20 +169,20 @@ def qute_history(url): def history_iter(start_time, reverse=False): """Iterate through the history and get items we're interested. - Keyword arguments: - reverse -- whether to reverse the history_dict before iterating. - start_time -- select history starting from this timestamp. + Arguments: + reverse -- whether to reverse the history_dict before iterating. + start_time -- select history starting from this timestamp. """ history = objreg.get('web-history').history_dict.values() if reverse: history = reversed(history) - end_time = start_time - 86400.0 # end is 24hrs earlier than start + end_time = start_time - 24*60*60 # end is 24hrs earlier than start for item in history: # Abort/continue as early as possible item_newer = item.atime > start_time - item_older = item.atime < end_time + item_older = item.atime <= end_time if reverse: # history_dict is reversed, we are going back in history. # so: @@ -202,12 +202,9 @@ def qute_history(url): if item_newer: return - # Skip items not within start_time and end_time # Skip redirects # Skip qute:// links - is_in_window = item.atime > end_time and item.atime <= start_time - is_internal = item.url.scheme() == 'qute' - if item.redirect or is_internal or not is_in_window: + if item.redirect or item.url.scheme() == 'qute': continue # Use item's url as title if there's no title. @@ -217,7 +214,7 @@ def qute_history(url): yield {"url": item_url, "title": item_title, "time": item_time} - if QUrl(url).path() == '/data': + if url.path() == '/data': # Use start_time in query or current time. start_time = QUrlQuery(url).queryItemValue("start_time") start_time = float(start_time) if start_time else time.time() diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 6cdeb13f2..d46626077 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -92,4 +92,4 @@ class TestHistoryHandler: fake_web_history._add_entry(entry) url = QUrl("qute://history/data?start_time={}".format(self.now)) - _mimetype, data = benchmark(qutescheme.qute_history, url) + _mimetype, _data = benchmark(qutescheme.qute_history, url) From c4416c8ac080d75298057a0140e19a7e700f74a0 Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Sun, 26 Feb 2017 21:56:24 +0500 Subject: [PATCH 04/21] Prevent crash with invalid start_time param. --- qutebrowser/browser/qutescheme.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 6308e20fe..b6b95de53 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -216,8 +216,11 @@ def qute_history(url): if url.path() == '/data': # Use start_time in query or current time. - start_time = QUrlQuery(url).queryItemValue("start_time") - start_time = float(start_time) if start_time else time.time() + try: + start_time = QUrlQuery(url).queryItemValue("start_time") + start_time = float(start_time) if start_time else time.time() + except ValueError as e: + raise QuteSchemeError("Query parameter start_time is invalid", e) if sys.hexversion >= 0x03050000: # On Python >= 3.5 we can reverse the ordereddict in-place and thus From 783769d302235c8af758499aebdf320b6cf883d9 Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Mon, 27 Feb 2017 21:20:54 +0500 Subject: [PATCH 05/21] Load new history items from next item's time. --- qutebrowser/browser/qutescheme.py | 22 +++++++++++++----- qutebrowser/html/history.html | 33 ++++++++++++++++++++++++--- tests/unit/browser/test_qutescheme.py | 29 +++++++++++++++++++---- 3 files changed, 70 insertions(+), 14 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index b6b95de53..06417518c 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -179,8 +179,17 @@ def qute_history(url): end_time = start_time - 24*60*60 # end is 24hrs earlier than start + # when history_dict is not reversed, we need to keep track of last item + # so that we can yield its atime + last_item = None + for item in history: - # Abort/continue as early as possible + # Skip redirects + # Skip qute:// links + if item.redirect or item.url.scheme() == 'qute': + continue + + # Skip items out of time window item_newer = item.atime > start_time item_older = item.atime <= end_time if reverse: @@ -189,6 +198,7 @@ def qute_history(url): # abort if item is older than start_time+24hr # skip if item is newer than start if item_older: + yield {"next": int(item.atime)} return if item_newer: continue @@ -198,15 +208,12 @@ def qute_history(url): # abort if item is newer than start_time # skip if item is older than start_time+24hrs if item_older: + last_item = item continue if item_newer: + yield {"next": int(last_item.atime)} return - # Skip redirects - # Skip qute:// links - if item.redirect or item.url.scheme() == 'qute': - continue - # Use item's url as title if there's no title. item_url = item.url.toDisplayString() item_title = item.title if item.title else item_url @@ -214,6 +221,9 @@ def qute_history(url): yield {"url": item_url, "title": item_title, "time": item_time} + # if we reached here, we had reached the end of history + yield {"next": -1} + if url.path() == '/data': # Use start_time in query or current time. try: diff --git a/qutebrowser/html/history.html b/qutebrowser/html/history.html index ba64316e4..9523b10de 100644 --- a/qutebrowser/html/history.html +++ b/qutebrowser/html/history.html @@ -28,6 +28,19 @@ table { text-align: left; } +#load { + color: #555; + font-weight: bold; + text-decoration: none; +} + +#eof { + color: #aaa; + margin-bottom: 30px; + text-align: center; + width: 100%; +} + .session-separator { color: #aaa; height: 40px; @@ -43,6 +56,8 @@ table { var global = { // The last history item that was seen. lastItem: null, + // The next time to load + nextTime: null, // The cutoff interval for session-separator (30 minutes) SESSION_CUTOFF: 30*60 }; @@ -144,14 +159,25 @@ var getJSON = function(url, callback) { */ var loadHistory = function() { url = "qute://history/data"; - if (global.lastItem !== null) { - startTime = parseInt(global.lastItem.time) - 1; + if (global.nextTime !== null) { + startTime = global.nextTime; url = "qute://history/data?start_time=" + startTime.toString(); } getJSON(url, function(status, history) { if (history !== undefined) { for (item of history) { + if (item.next === -1) { + // Reached end of history + window.onscroll = null; + document.getElementById('eof').style.display = "block" + document.getElementById('load').style.display = "none" + continue; + } else if (item.next !== undefined) { + global.nextTime = parseInt(item.next); + continue; + } + atime = new Date(parseInt(item.time)*1000); var session = getSessionNode(atime); var row = makeHistoryRow(item.url, item.title, atime.toLocaleTimeString()); @@ -166,10 +192,11 @@ var loadHistory = function() { {% block content %}

Browsing history

+ +Show more + {% endblock %} diff --git a/qutebrowser/javascript/history.js b/qutebrowser/javascript/history.js new file mode 100644 index 000000000..b4e584bb0 --- /dev/null +++ b/qutebrowser/javascript/history.js @@ -0,0 +1,138 @@ +/** + * Container for global stuff + */ +var global = { + // The last history item that was seen. + lastItem: null, + // The next time to load + nextTime: null, + // The cutoff interval for session-separator (30 minutes) + SESSION_CUTOFF: 30*60 +}; + +/** + * Finds or creates the session table>tbody to which item with given date + * should be added. + * + * @param {Date} date - the date of the item being added. + */ +var getSessionNode = function(date) { + var histContainer = document.getElementById('hist-container'); + + // Find/create table + var tableId = "hist-" + date.getDate() + date.getMonth() + date.getYear(); + var table = document.getElementById(tableId); + if (table === null) { + table = document.createElement("table"); + table.id = tableId; + + caption = document.createElement("caption"); + caption.className = "date"; + var options = {weekday: 'long', year: 'numeric', month: 'long', day: 'numeric'}; + caption.innerHTML = date.toLocaleDateString('en-US', options); + table.appendChild(caption); + + // Add table to page + histContainer.appendChild(table); + } + + // Find/create tbody + var tbody = table.lastChild; + if (tbody.tagName !== "TBODY") { // this is the caption + tbody = document.createElement("tbody"); + table.appendChild(tbody); + } + + // Create session-separator and new tbody if necessary + if (tbody.lastChild !== null && global.lastItem !== null) { + lastItemDate = new Date(parseInt(global.lastItem.time)*1000); + var interval = (lastItemDate.getTime() - date.getTime())/1000.00; + if (interval > global.SESSION_CUTOFF) { + // Add session-separator + var sessionSeparator = document.createElement('td'); + sessionSeparator.className = "session-separator"; + sessionSeparator.colSpan = 2; + sessionSeparator.innerHTML = "§"; + table.appendChild(document.createElement('tr')); + table.lastChild.appendChild(sessionSeparator); + + // Create new tbody + tbody = document.createElement("tbody"); + table.appendChild(tbody); + } + } + + return tbody; +} + +/** + * Given a history item, create and return for it. + * param {string} itemUrl - The url for this item + * param {string} itemTitle - The title for this item + * param {string} itemTime - The formatted time for this item + */ +var makeHistoryRow = function(itemUrl, itemTitle, itemTime) { + var row = document.createElement('tr'); + + var title = document.createElement('td'); + title.className = "title"; + var link = document.createElement('a'); + link.href = itemUrl; + link.innerHTML = itemTitle; + title.appendChild(link); + + var time = document.createElement('td'); + time.className = "time"; + time.innerHTML = itemTime; + + row.appendChild(title); + row.appendChild(time); + + return row; +} + +var getJSON = function(url, callback) { + var xhr = new XMLHttpRequest(); + xhr.open('GET', url, true); + xhr.responseType = 'json'; + xhr.onload = function() { + var status = xhr.status; + callback(status, xhr.response); + }; + xhr.send(); +}; + +/** + * Load new history. + */ +var loadHistory = function() { + url = "qute://history/data"; + if (global.nextTime !== null) { + startTime = global.nextTime; + url = "qute://history/data?start_time=" + startTime.toString(); + } + + getJSON(url, function(status, history) { + if (history !== undefined) { + for (item of history) { + if (item.next === -1) { + // Reached end of history + window.onscroll = null; + document.getElementById('eof').style.display = "block"; + document.getElementById('load').style.display = "none"; + continue; + } else if (item.next !== undefined) { + global.nextTime = parseInt(item.next); + continue; + } + + atime = new Date(parseInt(item.time)*1000); + var session = getSessionNode(atime); + var row = makeHistoryRow(item.url, item.title, atime.toLocaleTimeString()); + session.appendChild(row); + global.lastItem = item; + } + } + }); +} + From 3e45f739fc73c8b3d74ea6c22d7c2484af4d0adc Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Tue, 28 Feb 2017 17:21:42 +0500 Subject: [PATCH 08/21] Show message when Javascript is turned off. --- qutebrowser/html/history.html | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/qutebrowser/html/history.html b/qutebrowser/html/history.html index a7c0eafc5..75925f546 100644 --- a/qutebrowser/html/history.html +++ b/qutebrowser/html/history.html @@ -47,9 +47,22 @@ table { text-align: center; } +.error { + background-color: #ffbbbb; + border-radius: 5px; + font-weight: bold; + padding: 10px; + text-align: center; + width: 100%; + border: 1px solid #ff7777; +} + {% endblock %} {% block content %}

Browsing history

+
Show more From 9e6b0240f67d56e643bd6d11adc9b7e00149ec06 Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Tue, 28 Feb 2017 19:23:31 +0500 Subject: [PATCH 09/21] Put javascript in module, fix lint errors --- qutebrowser/html/history.html | 12 +- qutebrowser/javascript/history.js | 298 ++++++++++++++++++------------ 2 files changed, 186 insertions(+), 124 deletions(-) diff --git a/qutebrowser/html/history.html b/qutebrowser/html/history.html index 75925f546..33244896c 100644 --- a/qutebrowser/html/history.html +++ b/qutebrowser/html/history.html @@ -65,16 +65,22 @@ table {
-Show more +Show more {% endblock %} diff --git a/qutebrowser/javascript/history.js b/qutebrowser/javascript/history.js index b4e584bb0..3ff9f09db 100644 --- a/qutebrowser/javascript/history.js +++ b/qutebrowser/javascript/history.js @@ -1,138 +1,194 @@ /** - * Container for global stuff - */ -var global = { - // The last history item that was seen. - lastItem: null, - // The next time to load - nextTime: null, - // The cutoff interval for session-separator (30 minutes) - SESSION_CUTOFF: 30*60 -}; - -/** - * Finds or creates the session table>tbody to which item with given date - * should be added. + * Copyright 2017 Imran Sobir * - * @param {Date} date - the date of the item being added. + * This file is part of qutebrowser. + * + * qutebrowser is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * qutebrowser is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with qutebrowser. If not, see . */ -var getSessionNode = function(date) { - var histContainer = document.getElementById('hist-container'); - // Find/create table - var tableId = "hist-" + date.getDate() + date.getMonth() + date.getYear(); - var table = document.getElementById(tableId); - if (table === null) { - table = document.createElement("table"); - table.id = tableId; +"use strict"; - caption = document.createElement("caption"); - caption.className = "date"; - var options = {weekday: 'long', year: 'numeric', month: 'long', day: 'numeric'}; - caption.innerHTML = date.toLocaleDateString('en-US', options); - table.appendChild(caption); +window.loadHistory = (function() { + // The time of last history item. + var lastTime = null; - // Add table to page - histContainer.appendChild(table); - } + // The time to load next. + var nextTime = null; - // Find/create tbody - var tbody = table.lastChild; - if (tbody.tagName !== "TBODY") { // this is the caption - tbody = document.createElement("tbody"); - table.appendChild(tbody); - } + // The cutoff interval for session-separator (30 minutes) + var SESSION_CUTOFF = 30 * 60; - // Create session-separator and new tbody if necessary - if (tbody.lastChild !== null && global.lastItem !== null) { - lastItemDate = new Date(parseInt(global.lastItem.time)*1000); - var interval = (lastItemDate.getTime() - date.getTime())/1000.00; - if (interval > global.SESSION_CUTOFF) { - // Add session-separator - var sessionSeparator = document.createElement('td'); - sessionSeparator.className = "session-separator"; - sessionSeparator.colSpan = 2; - sessionSeparator.innerHTML = "§"; - table.appendChild(document.createElement('tr')); - table.lastChild.appendChild(sessionSeparator); + // The URL to fetch data from. + var DATA_URL = "qute://history/data"; - // Create new tbody + // Various fixed elements + var EOF_MESSAGE = document.getElementById("eof"); + var LOAD_LINK = document.getElementById("load"); + var HIST_CONTAINER = document.getElementById("hist-container"); + + /** + * Finds or creates the session table>tbody to which item with given date + * should be added. + * + * @param {Date} date - the date of the item being added. + * @returns {Element} the element to which new rows should be added. + */ + function getSessionNode(date) { + // Find/create table + var tableId = "hist-".concat(date.getDate(), date.getMonth(), + date.getYear()); + var table = document.getElementById(tableId); + if (table === null) { + table = document.createElement("table"); + table.id = tableId; + + // Caption contains human-readable date + var caption = document.createElement("caption"); + caption.className = "date"; + var options = { + "weekday": "long", + "year": "numeric", + "month": "long", + "day": "numeric", + }; + caption.innerHTML = date.toLocaleDateString("en-US", options); + table.appendChild(caption); + + // Add table to page + HIST_CONTAINER.appendChild(table); + } + + // Find/create tbody + var tbody = table.lastChild; + if (tbody.tagName !== "TBODY") { tbody = document.createElement("tbody"); table.appendChild(tbody); } - } - return tbody; -} + // Create session-separator and new tbody if necessary + if (tbody.lastChild !== null && lastTime !== null) { + var lastItemDate = new Date(lastTime * 1000); + var interval = (lastItemDate.getTime() - date.getTime()) / 1000.00; + if (interval > SESSION_CUTOFF) { + // Add session-separator + var sessionSeparator = document.createElement("td"); + sessionSeparator.className = "session-separator"; + sessionSeparator.colSpan = 2; + sessionSeparator.innerHTML = "§"; + table.appendChild(document.createElement("tr")); + table.lastChild.appendChild(sessionSeparator); -/** - * Given a history item, create and return for it. - * param {string} itemUrl - The url for this item - * param {string} itemTitle - The title for this item - * param {string} itemTime - The formatted time for this item - */ -var makeHistoryRow = function(itemUrl, itemTitle, itemTime) { - var row = document.createElement('tr'); - - var title = document.createElement('td'); - title.className = "title"; - var link = document.createElement('a'); - link.href = itemUrl; - link.innerHTML = itemTitle; - title.appendChild(link); - - var time = document.createElement('td'); - time.className = "time"; - time.innerHTML = itemTime; - - row.appendChild(title); - row.appendChild(time); - - return row; -} - -var getJSON = function(url, callback) { - var xhr = new XMLHttpRequest(); - xhr.open('GET', url, true); - xhr.responseType = 'json'; - xhr.onload = function() { - var status = xhr.status; - callback(status, xhr.response); - }; - xhr.send(); -}; - -/** - * Load new history. - */ -var loadHistory = function() { - url = "qute://history/data"; - if (global.nextTime !== null) { - startTime = global.nextTime; - url = "qute://history/data?start_time=" + startTime.toString(); - } - - getJSON(url, function(status, history) { - if (history !== undefined) { - for (item of history) { - if (item.next === -1) { - // Reached end of history - window.onscroll = null; - document.getElementById('eof').style.display = "block"; - document.getElementById('load').style.display = "none"; - continue; - } else if (item.next !== undefined) { - global.nextTime = parseInt(item.next); - continue; - } - - atime = new Date(parseInt(item.time)*1000); - var session = getSessionNode(atime); - var row = makeHistoryRow(item.url, item.title, atime.toLocaleTimeString()); - session.appendChild(row); - global.lastItem = item; + // Create new tbody + tbody = document.createElement("tbody"); + table.appendChild(tbody); } } - }); -} + return tbody; + } + + /** + * Given a history item, create and return for it. + * + * @param {string} itemUrl - The url for this item. + * @param {string} itemTitle - The title for this item. + * @param {string} itemTime - The formatted time for this item. + * @returns {Element} the completed tr. + */ + function makeHistoryRow(itemUrl, itemTitle, itemTime) { + var row = document.createElement("tr"); + + var title = document.createElement("td"); + title.className = "title"; + var link = document.createElement("a"); + link.href = itemUrl; + link.innerHTML = itemTitle; + title.appendChild(link); + + var time = document.createElement("td"); + time.className = "time"; + time.innerHTML = itemTime; + + row.appendChild(title); + row.appendChild(time); + + return row; + } + + /** + * Get JSON from given URL. + * + * @param {string} url - the url to fetch data from. + * @param {function} callback - the function to callback with data. + * @returns {void} + */ + function getJSON(url, callback) { + var xhr = new XMLHttpRequest(); + xhr.open("GET", url, true); + xhr.responseType = "json"; + xhr.onload = function() { + var status = xhr.status; + callback(status, xhr.response); + }; + xhr.send(); + } + + /** + * Receive history data from qute://history/data. + * + * @param {Number} status - The status of the query. + * @param {Array} history - History data. + * @returns {void} + */ + function receiveHistory(status, history) { + if (history === undefined) { + return; + } + + for (var i = 0, len = history.length - 1; i < len; i++) { + var item = history[i]; + var atime = new Date(item.time * 1000); + var session = getSessionNode(atime); + var row = makeHistoryRow(item.url, item.title, + atime.toLocaleTimeString()); + session.appendChild(row); + lastTime = item.time; + } + + var next = history[history.length - 1].next; + if (next === -1) { + // Reached end of history + window.onscroll = null; + EOF_MESSAGE.style.display = "block"; + LOAD_LINK.style.display = "none"; + } else { + nextTime = next; + } + } + + /** + * Load new history. + * @return {void} + */ + function loadHistory() { + if (nextTime === null) { + getJSON(DATA_URL, receiveHistory); + } else { + var url = DATA_URL.concat("?start_time=", nextTime.toString()); + getJSON(url, receiveHistory); + } + } + + return loadHistory; +})(); From cb6c6b814e066c389d4d6bfb3d2da6a919c52222 Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Tue, 28 Feb 2017 19:49:00 +0500 Subject: [PATCH 10/21] Fix pylint errors. --- qutebrowser/browser/qutescheme.py | 4 ++-- tests/unit/browser/test_qutescheme.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 77bb5fadc..0a09f4c9d 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -29,7 +29,7 @@ import sys import time import urllib.parse -from PyQt5.QtCore import QUrl, QUrlQuery +from PyQt5.QtCore import QUrlQuery import qutebrowser from qutebrowser.utils import (version, utils, jinja, log, message, docutils, @@ -203,7 +203,7 @@ def qute_history(url): if item_newer: continue else: - # history_dict is not reversed, we are going forward in history. + # history_dict isn't reversed, we are going forward in history. # so: # abort if item is newer than start_time # skip if item is older than start_time+24hrs diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 736e739aa..f5f517040 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -42,7 +42,7 @@ class TestJavascriptHandler: def patch_read_file(self, monkeypatch): """Patch utils.read_file to return few fake JS files.""" def _read_file(path, binary=False): - """Faked utils.read_file""" + """Faked utils.read_file.""" assert not binary for filename, content in self.js_files: if path == os.path.join('javascript', filename): @@ -75,13 +75,15 @@ class TestHistoryHandler: """Test the qute://history endpoint.""" + # Current time + now = time.time() + @pytest.fixture def entries(self): """Create fake history entries.""" # create 12 history items spaced 6 hours apart, starting from now entry_count = 12 interval = 6 * 60 * 60 - self.now = time.time() items = [] for i in range(entry_count): From 895620b536c50e6a47f43a784fa37ee994e6975c Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Thu, 2 Mar 2017 22:40:24 +0500 Subject: [PATCH 11/21] Don't assume 'next' appears last. --- qutebrowser/browser/qutescheme.py | 2 +- tests/unit/browser/test_qutescheme.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 0a09f4c9d..0ec3989d5 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -211,7 +211,7 @@ def qute_history(url): last_item = item continue if item_newer: - yield {"next": int(last_item.atime)} + yield {"next": int(last_item.atime if last_item else -1)} return # Use item's url as title if there's no title. diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index f5f517040..2fe5a5fc4 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -110,10 +110,10 @@ class TestHistoryHandler: fake_web_history.save() @pytest.mark.parametrize("start_time_offset, expected_item_count", [ - (0, 5), - (24*60*60, 5), - (48*60*60, 5), - (72*60*60, 1) + (0, 4), + (24*60*60, 4), + (48*60*60, 4), + (72*60*60, 0) ]) def test_qutehistory_data(self, start_time_offset, expected_item_count): """Ensure qute://history/data returns correct items.""" @@ -121,12 +121,13 @@ class TestHistoryHandler: url = QUrl("qute://history/data?start_time=" + str(start_time)) _mimetype, data = qutescheme.qute_history(url) items = json.loads(data) + items = [item for item in items if 'time' in item] # skip 'next' item assert len(items) == expected_item_count # test times end_time = start_time - 24*60*60 - for item in items[:expected_item_count-1]: + for item in items: assert item['time'] <= start_time assert item['time'] > end_time @@ -142,6 +143,7 @@ class TestHistoryHandler: url = QUrl("qute://history/data?start_time=" + str(start_time)) _mimetype, data = qutescheme.qute_history(url) items = json.loads(data) + items = [item for item in items if 'next' in item] # 'next' items if next_time == -1: assert items[-1]["next"] == -1 From 96e81f595fb021b5299124ce03e013a1df08ad40 Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Thu, 2 Mar 2017 23:14:00 +0500 Subject: [PATCH 12/21] Fix a case where 'next' is not correctly returned. --- qutebrowser/browser/qutescheme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 0ec3989d5..0ad71da36 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -222,7 +222,7 @@ def qute_history(url): yield {"url": item_url, "title": item_title, "time": item_time} # if we reached here, we had reached the end of history - yield {"next": -1} + yield {"next": int(last_item.atime if last_item else -1)} if url.path() == '/data': # Use start_time in query or current time. From 907d94591d2a0f90d9de60b92f5b67c00ac1cfa6 Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Thu, 2 Mar 2017 23:14:36 +0500 Subject: [PATCH 13/21] Make a now fixture to hold time of test. --- tests/unit/browser/test_qutescheme.py | 29 +++++++++++++++------------ 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 2fe5a5fc4..98dd4424e 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -75,11 +75,12 @@ class TestHistoryHandler: """Test the qute://history endpoint.""" - # Current time - now = time.time() + @pytest.fixture(scope="module") + def now(self): + return int(time.time()) @pytest.fixture - def entries(self): + def entries(self, now): """Create fake history entries.""" # create 12 history items spaced 6 hours apart, starting from now entry_count = 12 @@ -87,7 +88,7 @@ class TestHistoryHandler: items = [] for i in range(entry_count): - entry_atime = int(self.now - i * interval) + entry_atime = now - i * interval entry = history.Entry(atime=str(entry_atime), url=QUrl("www.x.com/" + str(i)), title="Page " + str(i)) items.insert(0, entry) @@ -115,9 +116,10 @@ class TestHistoryHandler: (48*60*60, 4), (72*60*60, 0) ]) - def test_qutehistory_data(self, start_time_offset, expected_item_count): + def test_qutehistory_data(self, start_time_offset, expected_item_count, + now): """Ensure qute://history/data returns correct items.""" - start_time = int(self.now) - start_time_offset + start_time = now - start_time_offset url = QUrl("qute://history/data?start_time=" + str(start_time)) _mimetype, data = qutescheme.qute_history(url) items = json.loads(data) @@ -137,26 +139,27 @@ class TestHistoryHandler: (48*60*60, -1), (72*60*60, -1) ]) - def test_qutehistory_next(self, start_time_offset, next_time): + def test_qutehistory_next(self, start_time_offset, next_time, now): """Ensure qute://history/data returns correct items.""" - start_time = int(self.now) - start_time_offset + start_time = now - start_time_offset url = QUrl("qute://history/data?start_time=" + str(start_time)) _mimetype, data = qutescheme.qute_history(url) items = json.loads(data) items = [item for item in items if 'next' in item] # 'next' items + assert len(items) == 1 if next_time == -1: - assert items[-1]["next"] == -1 + assert items[0]["next"] == -1 else: - assert items[-1]["next"] == int(self.now) - next_time + assert items[0]["next"] == now - next_time - def test_qute_history_benchmark(self, fake_web_history, benchmark): + def test_qute_history_benchmark(self, fake_web_history, benchmark, now): for t in range(100000): # one history per second entry = history.Entry( - atime=str(self.now - t), + atime=str(now - t), url=QUrl('www.x.com/{}'.format(t)), title='x at {}'.format(t)) fake_web_history._add_entry(entry) - url = QUrl("qute://history/data?start_time={}".format(self.now)) + url = QUrl("qute://history/data?start_time={}".format(now)) _mimetype, _data = benchmark(qutescheme.qute_history, url) From 0092b18c442b82fa571f07a5830d0d04a10e9d49 Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Sat, 4 Mar 2017 19:37:48 +0500 Subject: [PATCH 14/21] Fix qute:javascript on Windows. --- qutebrowser/browser/qutescheme.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 0ad71da36..697c6761b 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -25,6 +25,7 @@ Module attributes: """ import json +import os import sys import time import urllib.parse @@ -256,7 +257,8 @@ def qute_javascript(url): """ path = url.path() if path: - return 'text/html', utils.read_file("javascript" + path, binary=False) + path = "javascript" + os.sep.join(path.split('/')) + return 'text/html', utils.read_file(path, binary=False) else: raise QuteSchemeError("No file specified", ValueError()) From 3fbeecbec2da29a5d8fbd2570e6142ef42561476 Mon Sep 17 00:00:00 2001 From: Imran Sobir Date: Thu, 9 Mar 2017 21:35:22 +0500 Subject: [PATCH 15/21] Hide 'Show more' when Javascript is disabled. --- qutebrowser/html/history.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qutebrowser/html/history.html b/qutebrowser/html/history.html index 33244896c..d52a3c071 100644 --- a/qutebrowser/html/history.html +++ b/qutebrowser/html/history.html @@ -65,11 +65,13 @@ table {
-Show more +