diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 0f98a9216..e2e2d5107 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -158,6 +158,8 @@ def init(args, crash_handler): QDesktopServices.setUrlHandler('https', open_desktopservices_url) QDesktopServices.setUrlHandler('qute', open_desktopservices_url) + _import_history() + log.init.debug("Init done!") crash_handler.raise_crashdlg() @@ -474,6 +476,27 @@ def _init_modules(args, crash_handler): browsertab.init() +def _import_history(): + """Import a history text file into sqlite if it exists. + + In older versions of qutebrowser, history was stored in a text format. + This converts that file into the new sqlite format and removes it. + """ + path = os.path.join(standarddir.data(), 'history') + if not os.path.isfile(path): + return + + def action(): + with debug.log_time(log.init, 'Converting old history file to sqlite'): + objreg.get('web-history').read(path) + message.info('History import complete. Removing {}'.format(path)) + os.remove(path) + + # delay to give message time to appear before locking down for import + message.info('Converting {} to sqlite...'.format(path)) + QTimer.singleShot(100, action) + + class Quitter: """Utility class to quit/restart the QApplication. diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e7ff54037..8d10cdbbe 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -20,6 +20,7 @@ """Simple history which gets written to disk.""" import time +import os from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl @@ -70,37 +71,6 @@ class Entry: """Get the URL as a lossless string.""" return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) - @classmethod - def from_str(cls, line): - """Parse a history line like '12345 http://example.com title'.""" - data = line.split(maxsplit=2) - if len(data) == 2: - atime, url = data - title = "" - elif len(data) == 3: - atime, url, title = data - else: - raise ValueError("2 or 3 fields expected") - - url = QUrl(url) - if not url.isValid(): - raise ValueError("Invalid URL: {}".format(url.errorString())) - - # https://github.com/qutebrowser/qutebrowser/issues/670 - atime = atime.lstrip('\0') - - if '-' in atime: - atime, flags = atime.split('-') - else: - flags = '' - - if not set(flags).issubset('r'): - raise ValueError("Invalid flags {!r}".format(flags)) - - redirect = 'r' in flags - - return cls(atime, url, title, redirect=redirect) - class WebHistory(sql.SqlTable): @@ -199,6 +169,48 @@ class WebHistory(sql.SqlTable): entry = Entry(atime, url, title, redirect=redirect) self._add_entry(entry) + def _parse_entry(self, line): + """Parse a history line like '12345 http://example.com title'.""" + data = line.split(maxsplit=2) + if len(data) == 2: + atime, url = data + title = "" + elif len(data) == 3: + atime, url, title = data + else: + raise ValueError("2 or 3 fields expected") + + url = QUrl(url) + if not url.isValid(): + raise ValueError("Invalid URL: {}".format(url.errorString())) + + # https://github.com/qutebrowser/qutebrowser/issues/670 + atime = atime.lstrip('\0') + + if '-' in atime: + atime, flags = atime.split('-') + else: + flags = '' + + if not set(flags).issubset('r'): + raise ValueError("Invalid flags {!r}".format(flags)) + + redirect = 'r' in flags + + return (url, title, float(atime), bool(redirect)) + + def read(self, path): + """Import a text file into the sql database.""" + with open(path, 'r') as f: + rows = [] + for line in f: + try: + row = self._parse_entry(line.strip()) + rows.append(row) + except ValueError: + log.init.warning('Skipping history line {}'.format(line)) + self.insert_batch(rows) + def init(parent=None): """Initialize the web history. diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index eb1ad533c..d1d478302 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -55,16 +55,21 @@ def version(): return result.record().value(0) +def _prepare_query(querystr): + log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) + database = QSqlDatabase.database() + query = QSqlQuery(database) + query.prepare(querystr) + return query + + def run_query(querystr, values=None): """Run the given SQL query string on the database. Args: values: A list of positional parameter bindings. """ - log.sql.debug('Running SQL query: "{}"'.format(querystr)) - database = QSqlDatabase.database() - query = QSqlQuery(database) - query.prepare(querystr) + query = _prepare_query(querystr) for val in values or []: query.addBindValue(val) log.sql.debug('Query bindings: {}'.format(query.boundValues())) @@ -74,6 +79,28 @@ def run_query(querystr, values=None): return query +def run_batch(querystr, values): + """Run the given SQL query string on the database in batch mode. + + Args: + values: A list of lists, where each inner list contains positional + bindings for one run of the batch. + """ + query = _prepare_query(querystr) + transposed = [list(row) for row in zip(*values)] + for val in transposed: + query.addBindValue(val) + log.sql.debug('Batch Query bindings: {}'.format(query.boundValues())) + + db = QSqlDatabase.database() + db.transaction() + if not query.execBatch(): + raise SqlException('Failed to exec query "{}": "{}"'.format( + querystr, query.lastError().text())) + db.commit() + + return query + class SqlTable(QObject): """Interface to a sql table. @@ -102,8 +129,8 @@ class SqlTable(QObject): super().__init__(parent) self._name = name self._primary_key = primary_key - run_query("CREATE TABLE IF NOT EXISTS {} ({}, PRIMARY KEY ({}))" - .format(name, ','.join(fields), primary_key)) + run_query("CREATE TABLE IF NOT EXISTS {} ({})" + .format(name, ','.join(fields))) # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) @@ -172,6 +199,19 @@ class SqlTable(QObject): values) self.changed.emit() + def insert_batch(self, rows, replace=False): + """Performantly append multiple rows to the table. + + Args: + rows: A list of lists, where each sub-list is a row. + replace: If true, allow inserting over an existing primary key. + """ + cmd = "REPLACE" if replace else "INSERT" + paramstr = ','.join(['?'] * len(rows[0])) + run_batch("{} INTO {} values({})".format(cmd, self._name, paramstr), + rows) + self.changed.emit() + def delete_all(self): """Remove all row from the table.""" run_query("DELETE FROM {}".format(self._name)) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 9335cb239..e86950a82 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -148,88 +148,6 @@ def test_add_item_redirect_update(qtbot, tmpdir, hist): assert hist[url] == (url, '', 67890, True) -@pytest.mark.parametrize('line, expected', [ - ( - # old format without title - '12345 http://example.com/', - history.Entry(atime=12345, url=QUrl('http://example.com/'), title='',) - ), - ( - # trailing space without title - '12345 http://example.com/ ', - history.Entry(atime=12345, url=QUrl('http://example.com/'), title='',) - ), - ( - # new format with title - '12345 http://example.com/ this is a title', - history.Entry(atime=12345, url=QUrl('http://example.com/'), - title='this is a title') - ), - ( - # weird NUL bytes - '\x0012345 http://example.com/', - history.Entry(atime=12345, url=QUrl('http://example.com/'), title=''), - ), - ( - # redirect flag - '12345-r http://example.com/ this is a title', - history.Entry(atime=12345, url=QUrl('http://example.com/'), - title='this is a title', redirect=True) - ), -]) -def test_entry_parse_valid(line, expected): - entry = history.Entry.from_str(line) - assert entry == expected - - -@pytest.mark.parametrize('line', [ - '12345', # one field - '12345 ::', # invalid URL - 'xyz http://www.example.com/', # invalid timestamp - '12345-x http://www.example.com/', # invalid flags - '12345-r-r http://www.example.com/', # double flags -]) -def test_entry_parse_invalid(line): - with pytest.raises(ValueError): - history.Entry.from_str(line) - - -@hypothesis.given(strategies.text()) -def test_entry_parse_hypothesis(text): - """Make sure parsing works or gives us ValueError.""" - try: - history.Entry.from_str(text) - except ValueError: - pass - - -@pytest.mark.parametrize('entry, expected', [ - # simple - ( - history.Entry(12345, QUrl('http://example.com/'), "the title"), - "12345 http://example.com/ the title", - ), - # timestamp as float - ( - history.Entry(12345.678, QUrl('http://example.com/'), "the title"), - "12345 http://example.com/ the title", - ), - # no title - ( - history.Entry(12345.678, QUrl('http://example.com/'), ""), - "12345 http://example.com/", - ), - # redirect flag - ( - history.Entry(12345.678, QUrl('http://example.com/'), "", - redirect=True), - "12345-r http://example.com/", - ), -]) -def test_entry_str(entry, expected): - assert str(entry) == expected - - @pytest.fixture def hist_interface(): # pylint: disable=invalid-name @@ -298,3 +216,26 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): # For this to work, nothing can ever have called setDefaultInterface # before (so we need to test webengine before webkit) assert default_interface is None + + +def test_read(hist, tmpdir, caplog): + histfile = tmpdir / 'history' + histfile.write('''12345 http://example.com/ title + 12346 http://qutebrowser.org/ + 67890 http://example.com/path + + xyz http://example.com/bad-timestamp + 12345 + http://example.com/no-timestamp + 68891-r http://example.com/path/other + 68891-r-r http://example.com/double-flag''') + + with caplog.at_level(logging.WARNING): + hist.read(str(histfile)) + + assert list(hist) == [ + ('http://example.com/', 'title', 12345, False), + ('http://qutebrowser.org/', '', 12346, False), + ('http://example.com/path', '', 67890, False), + ('http://example.com/path/other', '', 68891, True) + ]