From c64b7d00e6a793f9d34af1ac294e53a626d7fe98 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 7 Jun 2017 15:57:56 +0200 Subject: [PATCH] Add separate table for history visits --- qutebrowser/browser/history.py | 29 ++++++++++++++++---- qutebrowser/completion/models/sqlcategory.py | 2 +- qutebrowser/completion/models/urlmodel.py | 4 +-- qutebrowser/misc/sql.py | 29 ++++++++++++++++---- 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index cb10c2bb2..718ad132d 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -72,15 +72,28 @@ class Entry: return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) +class HistoryVisits(sql.SqlTable): + + """Secondary table with visited URLs and timestamps.""" + + def __init__(self, parent=None): + super().__init__("Visits", ['url', 'atime'], + fkeys={'url': 'History(url)'}) + + class WebHistory(sql.SqlTable): """The global history of visited pages.""" def __init__(self, parent=None): - super().__init__("History", ['url', 'title', 'atime', 'redirect'], + super().__init__("History", + ['url', 'title', 'last_atime', 'redirect'], + constraints={'url': 'PRIMARY KEY'}, parent=parent) + self.visits = HistoryVisits(parent=self) self.create_index('HistoryIndex', 'url') self._contains_query = self.contains_query('url') + # FIXME self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' @@ -104,7 +117,8 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, int(entry.atime), - entry.redirect]) + entry.redirect], replace=True) + self.visits.insert([entry.url_str(), int(entry.atime)]) def get_recent(self): """Get the most recent history entries.""" @@ -216,7 +230,8 @@ class WebHistory(sql.SqlTable): redirect = 'r' in flags - return (url, title, float(atime), bool(redirect)) + return ((url, float(atime)), + (url, title, float(atime), bool(redirect))) def import_txt(self): """Import a history text file into sqlite if it exists. @@ -243,17 +258,20 @@ class WebHistory(sql.SqlTable): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: rows = [] + visit_rows = [] for (i, line) in enumerate(f): line = line.strip() if not line: continue try: - row = self._parse_entry(line.strip()) + visit_row, row = self._parse_entry(line.strip()) rows.append(row) + visit_rows.append(visit_row) except ValueError: raise Exception('Failed to parse line #{} of {}: "{}"' .format(i, path, line)) - self.insert_batch(rows) + self.insert_batch(rows, replace=True) + self.visits.insert_batch(visit_rows) @cmdutils.register(instance='web-history', debug=True) def debug_dump_history(self, dest): @@ -262,6 +280,7 @@ class WebHistory(sql.SqlTable): Args: dest: Where to write the file to. """ + # FIXME dest = os.path.expanduser(dest) lines = ('{}{} {} {}' diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index c3b5fd6c2..d363d09cb 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -30,7 +30,7 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, - select='*', where=None, group_by=None, parent=None): + select='*', where=None, group_by=None, suffix=None, parent=None): """Create a new completion category backed by a sql table. Args: diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index b106da366..1be364aba 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -75,9 +75,9 @@ def url(): columns_to_filter=[0, 1])) timefmt = config.get('completion', 'timestamp-format') - select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) + select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'History', sort_order='desc', sort_by='atime', + 'History', sort_order='desc', sort_by='last_atime', filter_fields=['url', 'title'], select='url, title, {}'.format(select_time), where='not redirect') model.add_category(hist_cat) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 0435d2ea3..dec051959 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -106,7 +106,8 @@ class SqlTable(QObject): changed = pyqtSignal() - def __init__(self, name, fields, parent=None): + def __init__(self, name, fields, constraints=None, fkeys=None, + parent=None): """Create a new table in the sql database. Raises SqlException if the table already exists. @@ -114,11 +115,23 @@ class SqlTable(QObject): Args: name: Name of the table. fields: A list of field names. + constraints: A dict mapping field names to constraint strings. + fkeys: A dict mapping field names to foreign keys. """ super().__init__(parent) self._name = name + + constraints = constraints or {} + fkeys = fkeys or {} + + column_defs = ['{} {}'.format(field, constraints.get(field, '')) + for field in fields] + for field, fkey in sorted(fkeys.items()): + column_defs.append('FOREIGN KEY({}) REFERENCES {}'.format( + field, fkey)) + q = Query("CREATE TABLE IF NOT EXISTS {} ({})" - .format(name, ','.join(fields))) + .format(name, ','.join(column_defs))) q.run() # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) @@ -171,25 +184,29 @@ class SqlTable(QObject): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, values): + def insert(self, values, replace=False): """Append a row to the table. Args: values: A list of values to insert. + replace: If set, replace existing values. """ paramstr = ','.join(['?'] * len(values)) - q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q = Query("INSERT {} INTO {} values({})".format( + 'OR REPLACE' if replace else '', self._name, paramstr)) q.run(values) self.changed.emit() - def insert_batch(self, rows): + def insert_batch(self, rows, replace=False): """Performantly append multiple rows to the table. Args: rows: A list of lists, where each sub-list is a row. + replace: If set, replace existing values. """ paramstr = ','.join(['?'] * len(rows[0])) - q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q = Query("INSERT {} INTO {} values({})".format( + 'OR REPLACE' if replace else '', self._name, paramstr)) transposed = [list(row) for row in zip(*rows)] for val in transposed: