Add debug-dump-history and fix sql history tests.

Trying to read from the sql database from another process was flaky.
This adds a debug-dump-history command which is used by the history BDD
tests to validate the history contents.

It outputs history in the old pre-SQL text format, so it might be
useful for those who want to manipulate their history as text.

qutebrowser/browser/history.py

@@ -19,6 +19,7 @@
 
 """Simple history which gets written to disk."""
 
+import os
 import time
 
 from PyQt5.QtCore import pyqtSlot, QUrl
@@ -233,6 +234,27 @@ class WebHistory(sql.SqlTable):
                                     .format(i, path, line))
         self.insert_batch(rows)
 
+    @cmdutils.register(instance='web-history', debug=True)
+    def debug_dump_history(self, dest):
+        """Dump the history to a file in the old pre-SQL format.
+
+        Args:
+            dest: Where to write the file to.
+        """
+        dest = os.path.expanduser(dest)
+
+        lines = ('{}{} {} {}'
+                 .format(int(x.atime), '-r' * x.redirect, x.url, x.title)
+                 for x in self.select(sort_by='atime', sort_order='asc'))
+
+        with open(dest, 'w', encoding='utf-8') as f:
+            try:
+                f.write('\n'.join(lines))
+            except OSError as e:
+                message.error('Could not write history: {}'.format(e))
+            else:
+                message.info("Dumped history to {}.".format(dest))
+
 
 def init(parent=None):
     """Initialize the web history.

qutebrowser/misc/sql.py

@@ -200,8 +200,14 @@ class SqlTable(QObject):
         run_query("DELETE FROM {}".format(self._name))
         self.changed.emit()
 
-    def select(self, sort_by, sort_order, limit):
+    def select(self, sort_by, sort_order, limit=-1):
-        """Remove all row from the table."""
+        """Remove all row from the table.
+
+        Args:
+            sort_by: name of column to sort by.
+            sort_order: 'asc' or 'desc'.
+            limit: max number of rows in result, defaults to -1 (unlimited).
+        """
         result = run_query('SELECT * FROM {} ORDER BY {} {} LIMIT {}'
                            .format(self._name, sort_by, sort_order, limit))
         while result.next():

tests/end2end/features/history.feature

@@ -11,49 +11,45 @@ Feature: Page history
     Scenario: Simple history saving
         When I open data/numbers/1.txt
         And I open data/numbers/2.txt
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/numbers/1.txt
             http://localhost:(port)/data/numbers/2.txt
 
     Scenario: History item with title
         When I open data/title.html
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/title.html Test title
 
     Scenario: History item with redirect
         When I open redirect-to?url=data/title.html without waiting
         And I wait until data/title.html is loaded
-        Then the history file should contain:
+        Then the history should contain:
             r http://localhost:(port)/redirect-to?url=data/title.html Test title
             http://localhost:(port)/data/title.html Test title
 
     Scenario: History item with spaces in URL
         When I open data/title with spaces.html
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/title%20with%20spaces.html Test title
 
     Scenario: History item with umlauts
         When I open data/äöü.html
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/%C3%A4%C3%B6%C3%BC.html Chäschüechli
 
-    # The following two tests use qute://history instead of checking the
-    # history file due to a race condition with sqlite.
-    # https://github.com/qutebrowser/qutebrowser/pull/2295#issuecomment-292786138
     @flaky @qtwebengine_todo: Error page message is not implemented
     Scenario: History with an error
         When I run :open file:///does/not/exist
         And I wait for "Error while loading file:///does/not/exist: Error opening /does/not/exist: *" in the log
-        And I open qute://history/data
+        Then the history should contain:
-        Then the page should contain the plaintext "Error loading page: file:///does/not/exist"
+            file:///does/not/exist Error loading page: file:///does/not/exist
 
     @qtwebengine_todo: Error page message is not implemented
     Scenario: History with a 404
         When I open status/404 without waiting
         And I wait for "Error while loading http://localhost:*/status/404: NOT FOUND" in the log
-        And I open qute://history/data
+        Then the history should contain:
-        Then the page should contain the plaintext "Error loading page: http://localhost:"
+            http://localhost:(port)/status/404 Error loading page: http://localhost:(port)/status/404
-        And the page should contain the plaintext "/status/404"
 
     Scenario: History with invalid URL
         When I run :tab-only
@@ -78,19 +74,19 @@ Feature: Page history
     Scenario: Clearing history
         When I open data/title.html
         And I run :history-clear --force
-        Then the history file should be empty
+        Then the history should be empty
 
     Scenario: Clearing history with confirmation
         When I open data/title.html
         And I run :history-clear
         And I wait for "Asking question <* title='Clear all browsing history?'>, *" in the log
         And I run :prompt-accept yes
-        Then the history file should be empty
+        Then the history should be empty
 
     Scenario: History with yanked URL and 'add to history' flag
         When I open data/hints/html/simple.html
         And I hint with args "--add-history links yank" and follow a
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/hints/html/simple.html Simple link
             http://localhost:(port)/data/hello.txt
 

tests/end2end/features/test_history_bdd.py

@@ -17,34 +17,33 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import os.path
+import re
+import tempfile
 
 import pytest_bdd as bdd
 
-from PyQt5.QtSql import QSqlDatabase
-
 bdd.scenarios('history.feature')
 
 
-@bdd.then(bdd.parsers.parse("the history file should contain:\n{expected}"))
+@bdd.then(bdd.parsers.parse("the history should contain:\n{expected}"))
-def check_history(quteproc, httpbin, expected):
+def check_history(quteproc, expected, httpbin):
-    path = os.path.join(quteproc.basedir, 'data', 'history.sqlite')
+    with tempfile.TemporaryDirectory() as tmpdir:
-    db = QSqlDatabase.addDatabase('QSQLITE')
+        path = os.path.join(tmpdir, 'history')
-    db.setDatabaseName(path)
+        quteproc.send_cmd(':debug-dump-history "{}"'.format(path))
-    assert db.open(), 'Failed to open history database'
+        quteproc.wait_for(category='message', loglevel=logging.INFO,
-    query = db.exec_('select * from History')
+                          message='Dumped history to {}.'.format(path))
-    actual = []
+
-    while query.next():
+        with open(path, 'r', encoding='utf-8') as f:
-        rec = query.record()
+            # ignore access times, they will differ in each run
-        url = rec.value(0)
+            actual = '\n'.join(re.sub('^\\d+-?', '', line).strip()
-        title = rec.value(1)
+                               for line in f.read().splitlines())
-        redirect = rec.value(3)
+
-        actual.append('{} {} {}'.format('r' * redirect, url, title).strip())
+    expected = expected.replace('(port)', str(httpbin.port))
-    db = None
+    assert actual == expected
-    QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName())
-    assert actual == expected.replace('(port)', str(httpbin.port)).splitlines()
 
 
-@bdd.then("the history file should be empty")
+@bdd.then("the history should be empty")
 def check_history_empty(quteproc, httpbin):
-    check_history(quteproc, httpbin, '')
+    check_history(quteproc, '', httpbin)

tests/unit/browser/webkit/test_history.py

@@ -261,3 +261,18 @@ def test_read_invalid(hist, tmpdir, line):
 
     with pytest.raises(Exception):
         hist.read(str(histfile))
+
+
+def test_debug_dump_history(hist, tmpdir):
+    hist.add_url(QUrl('http://example.com/1'), title="Title1", atime=12345)
+    hist.add_url(QUrl('http://example.com/2'), title="Title2", atime=12346)
+    hist.add_url(QUrl('http://example.com/3'), title="Title3", atime=12347)
+    hist.add_url(QUrl('http://example.com/4'), title="Title4", atime=12348,
+                 redirect=True)
+    histfile = tmpdir / 'history'
+    hist.debug_dump_history(str(histfile))
+    expected = ['12345 http://example.com/1 Title1',
+                '12346 http://example.com/2 Title2',
+                '12347 http://example.com/3 Title3',
+                '12348-r http://example.com/4 Title4']
+    assert histfile.read() == '\n'.join(expected)

tests/unit/misc/test_sql.py

@@ -53,7 +53,8 @@ def test_iter():
 @pytest.mark.parametrize('rows, sort_by, sort_order, limit, result', [
     ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', 5, [(1, 6), (2, 5), (3, 4)]),
     ([[2, 5], [1, 6], [3, 4]], 'a', 'desc', 3, [(3, 4), (2, 5), (1, 6)]),
-    ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)])
+    ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]),
+    ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', -1, [(1, 6), (2, 5), (3, 4)]),
 ])
 def test_select(rows, sort_by, sort_order, limit, result):
     table = sql.SqlTable('Foo', ['a', 'b'])