From 20000088dee5532c280fd741d665d3ab010f7a56 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 13 May 2017 21:32:47 -0400 Subject: [PATCH] Add debug-dump-history and fix sql history tests. Trying to read from the sql database from another process was flaky. This adds a debug-dump-history command which is used by the history BDD tests to validate the history contents. It outputs history in the old pre-SQL text format, so it might be useful for those who want to manipulate their history as text. --- qutebrowser/browser/history.py | 22 ++++++++++++ qutebrowser/misc/sql.py | 10 ++++-- tests/end2end/features/history.feature | 28 +++++++-------- tests/end2end/features/test_history_bdd.py | 41 +++++++++++----------- tests/unit/browser/webkit/test_history.py | 15 ++++++++ tests/unit/misc/test_sql.py | 3 +- 6 files changed, 79 insertions(+), 40 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 89f850b9d..e19463e75 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -19,6 +19,7 @@ """Simple history which gets written to disk.""" +import os import time from PyQt5.QtCore import pyqtSlot, QUrl @@ -233,6 +234,27 @@ class WebHistory(sql.SqlTable): .format(i, path, line)) self.insert_batch(rows) + @cmdutils.register(instance='web-history', debug=True) + def debug_dump_history(self, dest): + """Dump the history to a file in the old pre-SQL format. + + Args: + dest: Where to write the file to. + """ + dest = os.path.expanduser(dest) + + lines = ('{}{} {} {}' + .format(int(x.atime), '-r' * x.redirect, x.url, x.title) + for x in self.select(sort_by='atime', sort_order='asc')) + + with open(dest, 'w', encoding='utf-8') as f: + try: + f.write('\n'.join(lines)) + except OSError as e: + message.error('Could not write history: {}'.format(e)) + else: + message.info("Dumped history to {}.".format(dest)) + def init(parent=None): """Initialize the web history. diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 21af483c9..5c49767e8 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -200,8 +200,14 @@ class SqlTable(QObject): run_query("DELETE FROM {}".format(self._name)) self.changed.emit() - def select(self, sort_by, sort_order, limit): - """Remove all row from the table.""" + def select(self, sort_by, sort_order, limit=-1): + """Remove all row from the table. + + Args: + sort_by: name of column to sort by. + sort_order: 'asc' or 'desc'. + limit: max number of rows in result, defaults to -1 (unlimited). + """ result = run_query('SELECT * FROM {} ORDER BY {} {} LIMIT {}' .format(self._name, sort_by, sort_order, limit)) while result.next(): diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 6efd9ad0e..8174930f0 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -11,49 +11,45 @@ Feature: Page history Scenario: Simple history saving When I open data/numbers/1.txt And I open data/numbers/2.txt - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/numbers/1.txt http://localhost:(port)/data/numbers/2.txt Scenario: History item with title When I open data/title.html - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/title.html Test title Scenario: History item with redirect When I open redirect-to?url=data/title.html without waiting And I wait until data/title.html is loaded - Then the history file should contain: + Then the history should contain: r http://localhost:(port)/redirect-to?url=data/title.html Test title http://localhost:(port)/data/title.html Test title Scenario: History item with spaces in URL When I open data/title with spaces.html - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/title%20with%20spaces.html Test title Scenario: History item with umlauts When I open data/äöü.html - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/%C3%A4%C3%B6%C3%BC.html Chäschüechli - # The following two tests use qute://history instead of checking the - # history file due to a race condition with sqlite. - # https://github.com/qutebrowser/qutebrowser/pull/2295#issuecomment-292786138 @flaky @qtwebengine_todo: Error page message is not implemented Scenario: History with an error When I run :open file:///does/not/exist And I wait for "Error while loading file:///does/not/exist: Error opening /does/not/exist: *" in the log - And I open qute://history/data - Then the page should contain the plaintext "Error loading page: file:///does/not/exist" + Then the history should contain: + file:///does/not/exist Error loading page: file:///does/not/exist @qtwebengine_todo: Error page message is not implemented Scenario: History with a 404 When I open status/404 without waiting And I wait for "Error while loading http://localhost:*/status/404: NOT FOUND" in the log - And I open qute://history/data - Then the page should contain the plaintext "Error loading page: http://localhost:" - And the page should contain the plaintext "/status/404" + Then the history should contain: + http://localhost:(port)/status/404 Error loading page: http://localhost:(port)/status/404 Scenario: History with invalid URL When I run :tab-only @@ -78,19 +74,19 @@ Feature: Page history Scenario: Clearing history When I open data/title.html And I run :history-clear --force - Then the history file should be empty + Then the history should be empty Scenario: Clearing history with confirmation When I open data/title.html And I run :history-clear And I wait for "Asking question <* title='Clear all browsing history?'>, *" in the log And I run :prompt-accept yes - Then the history file should be empty + Then the history should be empty Scenario: History with yanked URL and 'add to history' flag When I open data/hints/html/simple.html And I hint with args "--add-history links yank" and follow a - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/hints/html/simple.html Simple link http://localhost:(port)/data/hello.txt diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 70f23f86b..197bc3d20 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -17,34 +17,33 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . +import logging import os.path +import re +import tempfile import pytest_bdd as bdd -from PyQt5.QtSql import QSqlDatabase - bdd.scenarios('history.feature') -@bdd.then(bdd.parsers.parse("the history file should contain:\n{expected}")) -def check_history(quteproc, httpbin, expected): - path = os.path.join(quteproc.basedir, 'data', 'history.sqlite') - db = QSqlDatabase.addDatabase('QSQLITE') - db.setDatabaseName(path) - assert db.open(), 'Failed to open history database' - query = db.exec_('select * from History') - actual = [] - while query.next(): - rec = query.record() - url = rec.value(0) - title = rec.value(1) - redirect = rec.value(3) - actual.append('{} {} {}'.format('r' * redirect, url, title).strip()) - db = None - QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) - assert actual == expected.replace('(port)', str(httpbin.port)).splitlines() +@bdd.then(bdd.parsers.parse("the history should contain:\n{expected}")) +def check_history(quteproc, expected, httpbin): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, 'history') + quteproc.send_cmd(':debug-dump-history "{}"'.format(path)) + quteproc.wait_for(category='message', loglevel=logging.INFO, + message='Dumped history to {}.'.format(path)) + + with open(path, 'r', encoding='utf-8') as f: + # ignore access times, they will differ in each run + actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() + for line in f.read().splitlines()) + + expected = expected.replace('(port)', str(httpbin.port)) + assert actual == expected -@bdd.then("the history file should be empty") +@bdd.then("the history should be empty") def check_history_empty(quteproc, httpbin): - check_history(quteproc, httpbin, '') + check_history(quteproc, '', httpbin) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index e6bbfcf69..16b86ca4a 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -261,3 +261,18 @@ def test_read_invalid(hist, tmpdir, line): with pytest.raises(Exception): hist.read(str(histfile)) + + +def test_debug_dump_history(hist, tmpdir): + hist.add_url(QUrl('http://example.com/1'), title="Title1", atime=12345) + hist.add_url(QUrl('http://example.com/2'), title="Title2", atime=12346) + hist.add_url(QUrl('http://example.com/3'), title="Title3", atime=12347) + hist.add_url(QUrl('http://example.com/4'), title="Title4", atime=12348, + redirect=True) + histfile = tmpdir / 'history' + hist.debug_dump_history(str(histfile)) + expected = ['12345 http://example.com/1 Title1', + '12346 http://example.com/2 Title2', + '12347 http://example.com/3 Title3', + '12348-r http://example.com/4 Title4'] + assert histfile.read() == '\n'.join(expected) diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index edc156a55..ca736b0ad 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -53,7 +53,8 @@ def test_iter(): @pytest.mark.parametrize('rows, sort_by, sort_order, limit, result', [ ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', 5, [(1, 6), (2, 5), (3, 4)]), ([[2, 5], [1, 6], [3, 4]], 'a', 'desc', 3, [(3, 4), (2, 5), (1, 6)]), - ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]) + ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]), + ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', -1, [(1, 6), (2, 5), (3, 4)]), ]) def test_select(rows, sort_by, sort_order, limit, result): table = sql.SqlTable('Foo', ['a', 'b'])