rnhmjoj/qutebrowser
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge commit '724e6b29c38a55f722f17997379f1ebe190fa6db' into imransobir/newhistory

Browse Source
This commit is contained in:
Florian Bruhin 2017-03-18 18:31:45 +01:00
commit 7c94b06be1
6 changed files with 451 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
qutebrowser/browser/qutescheme.py
View File

@ -24,14 +24,16 @@ Module attributes:
_HANDLERS: The handlers registered via decorators.
"""
import json
import os
import sys
import time
import datetime
import urllib.parse
from PyQt5.QtCore import QUrlQuery
import qutebrowser
from qutebrowser.config import config
from qutebrowser.utils import (version, utils, jinja, log, message, docutils,
objreg)
from qutebrowser.misc import objects
@ -165,83 +167,102 @@ def qute_bookmarks(_url):
@add_handler('history') # noqa
def qute_history(url):
"""Handler for qute:history. Display history."""
# Get current date from query parameter, if not given choose today.
curr_date = datetime.date.today()
try:
query_date = QUrlQuery(url).queryItemValue("date")
if query_date:
curr_date = datetime.datetime.strptime(query_date, "%Y-%m-%d")
curr_date = curr_date.date()
except ValueError:
log.misc.debug("Invalid date passed to qute:history: " + query_date)
"""Handler for qute:history. Display and serve history."""
def history_iter(start_time, reverse=False):
"""Iterate through the history and get items we're interested.
one_day = datetime.timedelta(days=1)
next_date = curr_date + one_day
prev_date = curr_date - one_day
def history_iter(reverse):
"""Iterate through the history and get items we're interested in."""
curr_timestamp = time.mktime(curr_date.timetuple())
Arguments:
reverse -- whether to reverse the history_dict before iterating.
start_time -- select history starting from this timestamp.
"""
history = objreg.get('web-history').history_dict.values()
if reverse:
history = reversed(history)
end_time = start_time - 24*60*60 # end is 24hrs earlier than start
# when history_dict is not reversed, we need to keep track of last item
# so that we can yield its atime
last_item = None
for item in history:
# If we can't apply the reverse performance trick below,
# at least continue as early as possible with old items.
# This gets us down from 550ms to 123ms with 500k old items on my
# machine.
if item.atime < curr_timestamp and not reverse:
continue
# Convert timestamp
try:
item_atime = datetime.datetime.fromtimestamp(item.atime)
except (ValueError, OSError, OverflowError):
log.misc.debug("Invalid timestamp {}.".format(item.atime))
continue
if reverse and item_atime.date() < curr_date:
# If we could reverse the history in-place, and this entry is
# older than today, only older entries will follow, so we can
# abort here.
return
# Skip items not on curr_date
# Skip redirects
# Skip qute:// links
is_internal = item.url.scheme() == 'qute'
is_not_today = item_atime.date() != curr_date
if item.redirect or is_internal or is_not_today:
if item.redirect or item.url.scheme() == 'qute':
continue
# Skip items out of time window
item_newer = item.atime > start_time
item_older = item.atime <= end_time
if reverse:
# history_dict is reversed, we are going back in history.
# so:
# abort if item is older than start_time+24hr
# skip if item is newer than start
if item_older:
yield {"next": int(item.atime)}
return
if item_newer:
continue
else:
# history_dict isn't reversed, we are going forward in history.
# so:
# abort if item is newer than start_time
# skip if item is older than start_time+24hrs
if item_older:
last_item = item
continue
if item_newer:
yield {"next": int(last_item.atime if last_item else -1)}
return
# Use item's url as title if there's no title.
item_url = item.url.toDisplayString()
item_title = item.title if item.title else item_url
display_atime = item_atime.strftime("%X")
item_time = int(item.atime * 1000)
yield (item_url, item_title, display_atime)
yield {"url": item_url, "title": item_title, "time": item_time}
if sys.hexversion >= 0x03050000:
# On Python >= 3.5 we can reverse the ordereddict in-place and thus
# apply an additional performance improvement in history_iter.
# On my machine, this gets us down from 550ms to 72us with 500k old
# items.
history = list(history_iter(reverse=True))
# if we reached here, we had reached the end of history
yield {"next": int(last_item.atime if last_item else -1)}
if url.path() == '/data':
# Use start_time in query or current time.
try:
start_time = QUrlQuery(url).queryItemValue("start_time")
start_time = float(start_time) if start_time else time.time()
except ValueError as e:
raise QuteSchemeError("Query parameter start_time is invalid", e)
if sys.hexversion >= 0x03050000:
# On Python >= 3.5 we can reverse the ordereddict in-place and thus
# apply an additional performance improvement in history_iter.
# On my machine, this gets us down from 550ms to 72us with 500k old
# items.
history = history_iter(start_time, reverse=True)
else:
# On Python 3.4, we can't do that, so we'd need to copy the entire
# history to a list. There, filter first and then reverse it here.
history = reversed(list(history_iter(start_time, reverse=False)))
return 'text/html', json.dumps(list(history))
else:
# On Python 3.4, we can't do that, so we'd need to copy the entire
# history to a list. There, filter first and then reverse it here.
history = reversed(list(history_iter(reverse=False)))
return 'text/html', jinja.render('history.html', title='History',
session_interval=config.get('ui', 'history-session-interval'))
html = jinja.render('history.html',
title='History',
history=history,
curr_date=curr_date,
next_date=next_date,
prev_date=prev_date,
today=datetime.date.today())
return 'text/html', html
@add_handler('javascript')
def qute_javascript(url):
"""Handler for qute:javascript.
Return content of file given as query parameter.
"""
path = url.path()
if path:
path = "javascript" + os.sep.join(path.split('/'))
return 'text/html', utils.read_file(path, binary=False)
else:
raise QuteSchemeError("No file specified", ValueError())
@add_handler('pyeval')

6
qutebrowser/config/configdata.py
View File

@ -292,6 +292,12 @@ def data(readonly=False):
)),
('ui', sect.KeyValue(
('history-session-interval',
SettingValue(typ.Int(), '30'),
"The maximum time in minutes between two history items for them "
"to be considered being from the same session. Use -1 to "
"disable separation."),
('zoom-levels',
SettingValue(typ.List(typ.Perc(minval=0)),
'25%,33%,50%,67%,75%,90%,100%,110%,125%,150%,175%,'

94
qutebrowser/html/history.html
View File

@ -16,43 +16,75 @@ td.time {
white-space: nowrap;
}
table {
margin-bottom: 30px;
}
.date {
color: #888;
font-size: 14pt;
padding-left: 25px;
}
.pagination-link {
display: inline-block;
margin-bottom: 10px;
margin-top: 10px;
padding-right: 10px;
}
.pagination-link > a {
color: #333;
color: #555;
font-size: 12pt;
padding-bottom: 15px;
font-weight: bold;
text-align: left;
}
{% endblock %}
#load {
color: #555;
font-weight: bold;
text-decoration: none;
}
#eof {
color: #aaa;
margin-bottom: 30px;
text-align: center;
width: 100%;
}
.session-separator {
color: #aaa;
height: 40px;
text-align: center;
}
.error {
background-color: #ffbbbb;
border-radius: 5px;
font-weight: bold;
padding: 10px;
text-align: center;
width: 100%;
border: 1px solid #ff7777;
}
{% endblock %}
{% block content %}
<h1>Browsing history</h1>
<noscript>
<div class="error">Javascript is required to view this page.</div>
</noscript>
<div id="hist-container"></div>
<span id="eof" style="display: none">end</span>
<a href="#" id="load" style="display: none">Show more</a>
<script type="text/javascript" src="qute://javascript/history.js"></script>
<script type="text/javascript">
window.SESSION_INTERVAL = {{session_interval}} * 60 * 1000;
<h1>Browsing history <span class="date">{{curr_date.strftime("%a, %d %B %Y")}}</span></h1>
window.onload = function() {
var loadLink = document.getElementById('load');
loadLink.style.display = null;
loadLink.addEventListener('click', function(ev) {
ev.preventDefault();
window.loadHistory();
});
<table>
<tbody>
{% for url, title, time in history %}
<tr>
<td class="title"><a href="{{url}}">{{title}}</a></td>
<td class="time">{{time}}</td>
</tr>
{% endfor %}
</tbody>
</table>
<span class="pagination-link"><a href="qute://history/?date={{prev_date.strftime("%Y-%m-%d")}}" rel="prev">Previous</a></span>
{% if today >= next_date %}
<span class="pagination-link"><a href="qute://history/?date={{next_date.strftime("%Y-%m-%d")}}" rel="next">Next</a></span>
{% endif %}
window.onscroll = function(ev) {
if ((window.innerHeight + window.scrollY) >= document.body.scrollHeight) {
window.loadHistory();
}
};
window.loadHistory();
};
</script>
{% endblock %}

190
qutebrowser/javascript/history.js Normal file
View File

@ -0,0 +1,190 @@
/**
* Copyright 2017 Imran Sobir
*
* This file is part of qutebrowser.
*
* qutebrowser is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* qutebrowser is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
window.loadHistory = (function() {
// Date of last seen item.
var lastItemDate = null;
// The time to load next.
var nextTime = null;
// The URL to fetch data from.
var DATA_URL = "qute://history/data";
// Various fixed elements
var EOF_MESSAGE = document.getElementById("eof");
var LOAD_LINK = document.getElementById("load");
var HIST_CONTAINER = document.getElementById("hist-container");
/**
* Finds or creates the session table>tbody to which item with given date
* should be added.
*
* @param {Date} date - the date of the item being added.
* @returns {Element} the element to which new rows should be added.
*/
function getSessionNode(date) {
// Find/create table
var tableId = ["hist", date.getDate(), date.getMonth(),
date.getYear()].join("-");
var table = document.getElementById(tableId);
if (table === null) {
table = document.createElement("table");
table.id = tableId;
// Caption contains human-readable date
var caption = document.createElement("caption");
caption.className = "date";
var options = {
"weekday": "long",
"year": "numeric",
"month": "long",
"day": "numeric",
};
caption.innerHTML = date.toLocaleDateString("en-US", options);
table.appendChild(caption);
// Add table to page
HIST_CONTAINER.appendChild(table);
}
// Find/create tbody
var tbody = table.lastChild;
if (tbody.tagName !== "TBODY") {
tbody = document.createElement("tbody");
table.appendChild(tbody);
}
// Create session-separator and new tbody if necessary
if (tbody.lastChild !== null && lastItemDate !== null &&
window.SESSION_INTERVAL > 0) {
var interval = lastItemDate.getTime() - date.getTime();
if (interval > window.SESSION_INTERVAL) {
// Add session-separator
var sessionSeparator = document.createElement("td");
sessionSeparator.className = "session-separator";
sessionSeparator.colSpan = 2;
sessionSeparator.innerHTML = "&#167;";
table.appendChild(document.createElement("tr"));
table.lastChild.appendChild(sessionSeparator);
// Create new tbody
tbody = document.createElement("tbody");
table.appendChild(tbody);
}
}
return tbody;
}
/**
* Given a history item, create and return <tr> for it.
*
* @param {string} itemUrl - The url for this item.
* @param {string} itemTitle - The title for this item.
* @param {string} itemTime - The formatted time for this item.
* @returns {Element} the completed tr.
*/
function makeHistoryRow(itemUrl, itemTitle, itemTime) {
var row = document.createElement("tr");
var title = document.createElement("td");
title.className = "title";
var link = document.createElement("a");
link.href = itemUrl;
link.innerHTML = itemTitle;
title.appendChild(link);
var time = document.createElement("td");
time.className = "time";
time.innerHTML = itemTime;
row.appendChild(title);
row.appendChild(time);
return row;
}
/**
* Get JSON from given URL.
*
* @param {string} url - the url to fetch data from.
* @param {function} callback - the function to callback with data.
* @returns {void}
*/
function getJSON(url, callback) {
var xhr = new XMLHttpRequest();
xhr.open("GET", url, true);
xhr.responseType = "json";
xhr.onload = function() {
var status = xhr.status;
callback(status, xhr.response);
};
xhr.send();
}
/**
* Receive history data from qute://history/data.
*
* @param {Number} status - The status of the query.
* @param {Array} history - History data.
* @returns {void}
*/
function receiveHistory(status, history) {
if (history === null) {
return;
}
for (var i = 0, len = history.length - 1; i < len; i++) {
var item = history[i];
var currentItemDate = new Date(item.time);
getSessionNode(currentItemDate).appendChild(makeHistoryRow(
item.url, item.title, currentItemDate.toLocaleTimeString()
));
lastItemDate = currentItemDate;
}
var next = history[history.length - 1].next;
if (next === -1) {
// Reached end of history
window.onscroll = null;
EOF_MESSAGE.style.display = "block";
LOAD_LINK.style.display = "none";
} else {
nextTime = next;
}
}
/**
* Load new history.
* @return {void}
*/
function loadHistory() {
if (nextTime === null) {
getJSON(DATA_URL, receiveHistory);
} else {
var url = DATA_URL.concat("?start_time=", nextTime.toString());
getJSON(url, receiveHistory);
}
}
return loadHistory;
})();

2
tests/end2end/features/history.feature
View File

@ -77,7 +77,7 @@ Feature: Page history
Scenario: Listing history
When I open data/numbers/3.txt
And I open data/numbers/4.txt
And I open qute:history
And I open qute://history/data
Then the page should contain the plaintext "3.txt"
Then the page should contain the plaintext "4.txt"

192
tests/unit/browser/test_qutescheme.py
View File

@ -17,8 +17,9 @@
# You should have received a copy of the GNU General Public License
# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
import datetime
import collections
import json
import os
import time
from PyQt5.QtCore import QUrl
import pytest
@ -27,30 +28,72 @@ from qutebrowser.browser import history, qutescheme
from qutebrowser.utils import objreg
Dates = collections.namedtuple('Dates', ['yesterday', 'today', 'tomorrow'])
class TestJavascriptHandler:
"""Test the qute://javascript endpoint."""
# Tuples of fake JS files and their content.
js_files = [
('foo.js', "var a = 'foo';"),
('bar.js', "var a = 'bar';"),
]
@pytest.fixture(autouse=True)
def patch_read_file(self, monkeypatch):
"""Patch utils.read_file to return few fake JS files."""
def _read_file(path, binary=False):
"""Faked utils.read_file."""
assert not binary
for filename, content in self.js_files:
if path == os.path.join('javascript', filename):
return content
raise OSError("File not found {}!".format(path))
monkeypatch.setattr('qutebrowser.utils.utils.read_file', _read_file)
@pytest.mark.parametrize("filename, content", js_files)
def test_qutejavascript(self, filename, content):
url = QUrl("qute://javascript/{}".format(filename))
_mimetype, data = qutescheme.qute_javascript(url)
assert data == content
def test_qutejavascript_404(self):
url = QUrl("qute://javascript/404.js")
with pytest.raises(qutescheme.QuteSchemeOSError):
qutescheme.data_for_url(url)
def test_qutejavascript_empty_query(self):
url = QUrl("qute://javascript")
with pytest.raises(qutescheme.QuteSchemeError):
qutescheme.qute_javascript(url)
class TestHistoryHandler:
"""Test the qute://history endpoint."""
@pytest.fixture
def dates(self):
one_day = datetime.timedelta(days=1)
today = datetime.datetime.today()
tomorrow = today + one_day
yesterday = today - one_day
return Dates(yesterday, today, tomorrow)
@pytest.fixture(scope="module")
def now(self):
return int(time.time())
@pytest.fixture
def entries(self, dates):
today = history.Entry(atime=str(dates.today.timestamp()),
url=QUrl('www.today.com'), title='today')
tomorrow = history.Entry(atime=str(dates.tomorrow.timestamp()),
url=QUrl('www.tomorrow.com'), title='tomorrow')
yesterday = history.Entry(atime=str(dates.yesterday.timestamp()),
url=QUrl('www.yesterday.com'), title='yesterday')
return Dates(yesterday, today, tomorrow)
def entries(self, now):
"""Create fake history entries."""
# create 12 history items spaced 6 hours apart, starting from now
entry_count = 12
interval = 6 * 60 * 60
items = []
for i in range(entry_count):
entry_atime = now - i * interval
entry = history.Entry(atime=str(entry_atime),
url=QUrl("www.x.com/" + str(i)), title="Page " + str(i))
items.insert(0, entry)
return items
@pytest.fixture
def fake_web_history(self, fake_save_manager, tmpdir):
@ -62,78 +105,61 @@ class TestHistoryHandler:
@pytest.fixture(autouse=True)
def fake_history(self, fake_web_history, entries):
"""Create fake history for three different days."""
fake_web_history._add_entry(entries.yesterday)
fake_web_history._add_entry(entries.today)
fake_web_history._add_entry(entries.tomorrow)
"""Create fake history."""
for item in entries:
fake_web_history._add_entry(item)
fake_web_history.save()
def test_history_without_query(self):
"""Ensure qute://history shows today's history without any query."""
_mimetype, data = qutescheme.qute_history(QUrl("qute://history"))
key = "<span class=\"date\">{}</span>".format(
datetime.date.today().strftime("%a, %d %B %Y"))
assert key in data
def test_history_with_bad_query(self):
"""Ensure qute://history shows today's history with bad query."""
url = QUrl("qute://history?date=204-blaah")
@pytest.mark.parametrize("start_time_offset, expected_item_count", [
(0, 4),
(24*60*60, 4),
(48*60*60, 4),
(72*60*60, 0)
])
def test_qutehistory_data(self, start_time_offset, expected_item_count,
now):
"""Ensure qute://history/data returns correct items."""
start_time = now - start_time_offset
url = QUrl("qute://history/data?start_time=" + str(start_time))
_mimetype, data = qutescheme.qute_history(url)
key = "<span class=\"date\">{}</span>".format(
datetime.date.today().strftime("%a, %d %B %Y"))
assert key in data
items = json.loads(data)
items = [item for item in items if 'time' in item] # skip 'next' item
def test_history_today(self):
"""Ensure qute://history shows history for today."""
url = QUrl("qute://history")
assert len(items) == expected_item_count
# test times
end_time = start_time - 24*60*60
for item in items:
assert item['time'] <= start_time * 1000
assert item['time'] > end_time * 1000
@pytest.mark.parametrize("start_time_offset, next_time", [
(0, 24*60*60),
(24*60*60, 48*60*60),
(48*60*60, -1),
(72*60*60, -1)
])
def test_qutehistory_next(self, start_time_offset, next_time, now):
"""Ensure qute://history/data returns correct items."""
start_time = now - start_time_offset
url = QUrl("qute://history/data?start_time=" + str(start_time))
_mimetype, data = qutescheme.qute_history(url)
assert "today" in data
assert "tomorrow" not in data
assert "yesterday" not in data
items = json.loads(data)
items = [item for item in items if 'next' in item] # 'next' items
assert len(items) == 1
def test_history_yesterday(self, dates):
"""Ensure qute://history shows history for yesterday."""
url = QUrl("qute://history?date=" +
dates.yesterday.strftime("%Y-%m-%d"))
_mimetype, data = qutescheme.qute_history(url)
assert "today" not in data
assert "tomorrow" not in data
assert "yesterday" in data
if next_time == -1:
assert items[0]["next"] == -1
else:
assert items[0]["next"] == now - next_time
def test_history_tomorrow(self, dates):
"""Ensure qute://history shows history for tomorrow."""
url = QUrl("qute://history?date=" +
dates.tomorrow.strftime("%Y-%m-%d"))
_mimetype, data = qutescheme.qute_history(url)
assert "today" not in data
assert "tomorrow" in data
assert "yesterday" not in data
def test_no_next_link_to_future(self, dates):
"""Ensure there's no next link pointing to the future."""
url = QUrl("qute://history")
_mimetype, data = qutescheme.qute_history(url)
assert "Next" not in data
url = QUrl("qute://history?date=" +
dates.tomorrow.strftime("%Y-%m-%d"))
_mimetype, data = qutescheme.qute_history(url)
assert "Next" not in data
def test_qute_history_benchmark(self, dates, entries, fake_web_history,
benchmark):
for i in range(100000):
def test_qute_history_benchmark(self, fake_web_history, benchmark, now):
for t in range(100000): # one history per second
entry = history.Entry(
atime=str(dates.yesterday.timestamp()),
url=QUrl('www.yesterday.com/{}'.format(i)),
title='yesterday')
atime=str(now - t),
url=QUrl('www.x.com/{}'.format(t)),
title='x at {}'.format(t))
fake_web_history._add_entry(entry)
fake_web_history._add_entry(entries.today)
fake_web_history._add_entry(entries.tomorrow)
url = QUrl("qute://history")
_mimetype, data = benchmark(qutescheme.qute_history, url)
assert "today" in data
assert "tomorrow" not in data
assert "yesterday" not in data
url = QUrl("qute://history/data?start_time={}".format(now))
_mimetype, _data = benchmark(qutescheme.qute_history, url)