Add LIMIT to history query.

For performance, re-introduce web-history-max-items.
As the history query has now become a very specific multi-part query and
history completion was the only consumer of SqlCategory, SqlCategory is
now replaced by a HistoryCategory class.
This commit is contained in:
Ryan Roden-Corrent 2017-07-14 09:28:06 -04:00
parent 8745f80d90
commit c32d452786
7 changed files with 204 additions and 223 deletions

View File

@ -17,7 +17,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>. # along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
"""A completion model backed by SQL tables.""" """A completion category that queries the SQL History store."""
import re import re
@ -26,50 +26,50 @@ from PyQt5.QtSql import QSqlQueryModel
from qutebrowser.misc import sql from qutebrowser.misc import sql
from qutebrowser.utils import debug from qutebrowser.utils import debug
from qutebrowser.commands import cmdexc from qutebrowser.commands import cmdexc
from qutebrowser.config import config
class SqlCategory(QSqlQueryModel): class HistoryCategory(QSqlQueryModel):
"""Wraps a SqlQuery for use as a completion category.""" """A completion category that queries the SQL History store."""
def __init__(self, name, *, title=None, filter_fields, sort_by=None, def __init__(self, *, delete_func=None, parent=None):
sort_order=None, select='*', """Create a new History completion category."""
delete_func=None, parent=None):
"""Create a new completion category backed by a sql table.
Args:
name: Name of the table in the database.
title: Title of category, defaults to table name.
filter_fields: Names of fields to apply filter pattern to.
select: A custom result column expression for the select statement.
sort_by: The name of the field to sort by, or None for no sorting.
sort_order: Either 'asc' or 'desc', if sort_by is non-None
delete_func: Callback to delete a selected item.
"""
super().__init__(parent=parent) super().__init__(parent=parent)
self.name = title or name self.name = "History"
querystr = 'select {} from {} where ('.format(select, name) # replace ' to avoid breaking the query
timefmt = "strftime('{}', last_atime, 'unixepoch')".format(
config.get('completion', 'timestamp-format').replace("'", "`"))
self._query = sql.Query(' '.join([
"SELECT url, title, {}".format(timefmt),
"FROM CompletionHistory",
# the incoming pattern will have literal % and _ escaped with '\' # the incoming pattern will have literal % and _ escaped with '\'
# we need to tell sql to treat '\' as an escape character # we need to tell sql to treat '\' as an escape character
querystr += ' or '.join("{} like :pattern escape '\\'".format(f) "WHERE (url LIKE :pat escape '\\' or title LIKE :pat escape '\\')",
for f in filter_fields) self._atime_expr(),
querystr += ')' "ORDER BY last_atime DESC",
]), forward_only=False)
if sort_by: # advertise that this model filters by URL and title
assert sort_order in ['asc', 'desc'], sort_order self.columns_to_filter = [0, 1]
querystr += ' order by {} {}'.format(sort_by, sort_order)
else:
assert sort_order is None, sort_order
self._query = sql.Query(querystr, forward_only=False)
# map filter_fields to indices
col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name))
rec = col_query.run().record()
self.columns_to_filter = [rec.indexOf(n) for n in filter_fields]
self.delete_func = delete_func self.delete_func = delete_func
def _atime_expr(self):
"""If max_items is set, return an expression to limit the query."""
max_items = config.get('completion', 'web-history-max-items')
if max_items < 0:
return ''
min_atime = sql.Query(' '.join([
'SELECT min(last_atime) FROM',
'(SELECT last_atime FROM CompletionHistory',
'ORDER BY last_atime DESC LIMIT :limit)',
])).run(limit=max_items).value()
return "AND last_atime >= {}".format(min_atime)
def set_pattern(self, pattern): def set_pattern(self, pattern):
"""Set the pattern used to filter results. """Set the pattern used to filter results.
@ -83,7 +83,7 @@ class SqlCategory(QSqlQueryModel):
pattern = re.sub(r' +', '%', pattern) pattern = re.sub(r' +', '%', pattern)
pattern = '%{}%'.format(pattern) pattern = '%{}%'.format(pattern)
with debug.log_time('sql', 'Running completion query'): with debug.log_time('sql', 'Running completion query'):
self._query.run(pattern=pattern) self._query.run(pat=pattern)
self.setQuery(self._query) self.setQuery(self._query)
def delete_cur_item(self, index): def delete_cur_item(self, index):

View File

@ -20,8 +20,7 @@
"""Function to return the url completion model for the `open` command.""" """Function to return the url completion model for the `open` command."""
from qutebrowser.completion.models import (completionmodel, listcategory, from qutebrowser.completion.models import (completionmodel, listcategory,
sqlcategory) histcategory)
from qutebrowser.config import config
from qutebrowser.utils import log, objreg from qutebrowser.utils import log, objreg
@ -66,14 +65,6 @@ def url():
model.add_category(listcategory.ListCategory( model.add_category(listcategory.ListCategory(
'Bookmarks', bookmarks, delete_func=_delete_bookmark)) 'Bookmarks', bookmarks, delete_func=_delete_bookmark))
# replace 's to avoid breaking the query hist_cat = histcategory.HistoryCategory(delete_func=_delete_history)
timefmt = config.get('completion', 'timestamp-format').replace("'", "`")
select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt)
hist_cat = sqlcategory.SqlCategory(
'CompletionHistory', title='History',
sort_order='desc', sort_by='last_atime',
filter_fields=['url', 'title'],
select='url, title, {}'.format(select_time),
delete_func=_delete_history)
model.add_category(hist_cat) model.add_category(hist_cat)
return model return model

View File

@ -382,7 +382,6 @@ class ConfigManager(QObject):
('storage', 'offline-storage-default-quota'), ('storage', 'offline-storage-default-quota'),
('storage', 'offline-web-application-cache-quota'), ('storage', 'offline-web-application-cache-quota'),
('content', 'css-regions'), ('content', 'css-regions'),
('completion', 'web-history-max-items'),
] ]
CHANGED_OPTIONS = { CHANGED_OPTIONS = {
('content', 'cookies-accept'): ('content', 'cookies-accept'):

View File

@ -502,6 +502,11 @@ def data(readonly=False):
"How many commands to save in the command history.\n\n" "How many commands to save in the command history.\n\n"
"0: no history / -1: unlimited"), "0: no history / -1: unlimited"),
('web-history-max-items',
SettingValue(typ.Int(minval=-1), '-1'),
"How many URLs to show in the web history.\n\n"
"0: no history / -1: unlimited"),
('quick-complete', ('quick-complete',
SettingValue(typ.Bool(), 'true'), SettingValue(typ.Bool(), 'true'),
"Whether to move on to the next part when there's only one " "Whether to move on to the next part when there's only one "

View File

@ -0,0 +1,150 @@
# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) <ryan@rcorre.net>
#
# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
"""Test the web history completion category."""
import unittest.mock
import time
import pytest
from qutebrowser.misc import sql
from qutebrowser.completion.models import histcategory
from qutebrowser.commands import cmdexc
@pytest.fixture
def hist(init_sql, config_stub):
config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d',
'web-history-max-items': -1}
return sql.SqlTable('CompletionHistory', ['url', 'title', 'last_atime'])
@pytest.mark.parametrize('pattern, before, after', [
('foo',
[('foo', ''), ('bar', ''), ('aafobbb', '')],
[('foo',)]),
('FOO',
[('foo', ''), ('bar', ''), ('aafobbb', '')],
[('foo',)]),
('foo',
[('FOO', ''), ('BAR', ''), ('AAFOBBB', '')],
[('FOO',)]),
('foo',
[('baz', 'bar'), ('foo', ''), ('bar', 'foo')],
[('foo', ''), ('bar', 'foo')]),
('foo',
[('fooa', ''), ('foob', ''), ('fooc', '')],
[('fooa', ''), ('foob', ''), ('fooc', '')]),
('foo',
[('foo', 'bar'), ('bar', 'foo'), ('biz', 'baz')],
[('foo', 'bar'), ('bar', 'foo')]),
('foo bar',
[('foo', ''), ('bar foo', ''), ('xfooyybarz', '')],
[('xfooyybarz', '')]),
('foo%bar',
[('foo%bar', ''), ('foo bar', ''), ('foobar', '')],
[('foo%bar', '')]),
('_',
[('a_b', ''), ('__a', ''), ('abc', '')],
[('a_b', ''), ('__a', '')]),
('%',
[('\\foo', '\\bar')],
[]),
("can't",
[("can't touch this", ''), ('a', '')],
[("can't touch this", '')]),
])
def test_set_pattern(pattern, before, after, model_validator, hist):
"""Validate the filtering and sorting results of set_pattern."""
for row in before:
hist.insert({'url': row[0], 'title': row[1], 'last_atime': 1})
cat = histcategory.HistoryCategory()
model_validator.set_model(cat)
cat.set_pattern(pattern)
model_validator.validate(after)
@pytest.mark.parametrize('max_items, before, after', [
(-1, [
('a', 'a', '2017-04-16'),
('b', 'b', '2017-06-16'),
('c', 'c', '2017-05-16'),
], [
('b', 'b', '2017-06-16'),
('c', 'c', '2017-05-16'),
('a', 'a', '2017-04-16'),
]),
(3, [
('a', 'a', '2017-04-16'),
('b', 'b', '2017-06-16'),
('c', 'c', '2017-05-16'),
], [
('b', 'b', '2017-06-16'),
('c', 'c', '2017-05-16'),
('a', 'a', '2017-04-16'),
]),
(2, [
('a', 'a', '2017-04-16'),
('b', 'b', '2017-06-16'),
('c', 'c', '2017-05-16'),
], [
('b', 'b', '2017-06-16'),
('c', 'c', '2017-05-16'),
])
])
def test_sorting(max_items, before, after, model_validator, hist, config_stub):
"""Validate the filtering and sorting results of set_pattern."""
config_stub.data['completion']['web-history-max-items'] = max_items
for url, title, atime in before:
timestamp = time.mktime(time.strptime(atime, '%Y-%m-%d'))
hist.insert({'url': url, 'title': title, 'last_atime': timestamp})
cat = histcategory.HistoryCategory()
model_validator.set_model(cat)
cat.set_pattern('')
model_validator.validate(after)
def test_delete_cur_item(hist):
hist.insert({'url': 'foo', 'title': 'Foo'})
hist.insert({'url': 'bar', 'title': 'Bar'})
func = unittest.mock.Mock(spec=[])
cat = histcategory.HistoryCategory(delete_func=func)
cat.set_pattern('')
cat.delete_cur_item(cat.index(0, 0))
func.assert_called_with(['foo', 'Foo', ''])
def test_delete_cur_item_no_func(hist):
hist.insert({'url': 'foo', 'title': 1})
hist.insert({'url': 'bar', 'title': 2})
cat = histcategory.HistoryCategory()
cat.set_pattern('')
with pytest.raises(cmdexc.CommandError, match='Cannot delete this item'):
cat.delete_cur_item(cat.index(0, 0))

View File

@ -137,8 +137,10 @@ def bookmarks(bookmark_manager_stub):
@pytest.fixture @pytest.fixture
def web_history(init_sql, stubs): def web_history(init_sql, stubs, config_stub):
"""Fixture which provides a web-history object.""" """Fixture which provides a web-history object."""
config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d',
'web-history-max-items': -1}
stub = history.WebHistory() stub = history.WebHistory()
objreg.register('web-history', stub) objreg.register('web-history', stub)
yield stub yield stub
@ -266,7 +268,7 @@ def test_bookmark_completion(qtmodeltester, bookmarks):
}) })
def test_url_completion(qtmodeltester, config_stub, web_history_populated, def test_url_completion(qtmodeltester, web_history_populated,
quickmarks, bookmarks): quickmarks, bookmarks):
"""Test the results of url completion. """Test the results of url completion.
@ -275,7 +277,6 @@ def test_url_completion(qtmodeltester, config_stub, web_history_populated,
- entries are sorted by access time - entries are sorted by access time
- only the most recent entry is included for each url - only the most recent entry is included for each url
""" """
config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'}
model = urlmodel.url() model = urlmodel.url()
model.set_pattern('') model.set_pattern('')
qtmodeltester.data_display_may_return_none = True qtmodeltester.data_display_may_return_none = True
@ -318,11 +319,10 @@ def test_url_completion(qtmodeltester, config_stub, web_history_populated,
('foo%bar', '', '%', 1), ('foo%bar', '', '%', 1),
('foobar', '', '%', 0), ('foobar', '', '%', 0),
]) ])
def test_url_completion_pattern(config_stub, web_history, def test_url_completion_pattern(web_history, quickmark_manager_stub,
quickmark_manager_stub, bookmark_manager_stub, bookmark_manager_stub, url, title, pattern,
url, title, pattern, rowcount): rowcount):
"""Test that url completion filters by url and title.""" """Test that url completion filters by url and title."""
config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'}
web_history.add_url(QUrl(url), title) web_history.add_url(QUrl(url), title)
model = urlmodel.url() model = urlmodel.url()
model.set_pattern(pattern) model.set_pattern(pattern)
@ -330,10 +330,9 @@ def test_url_completion_pattern(config_stub, web_history,
assert model.rowCount(model.index(2, 0)) == rowcount assert model.rowCount(model.index(2, 0)) == rowcount
def test_url_completion_delete_bookmark(qtmodeltester, config_stub, bookmarks, def test_url_completion_delete_bookmark(qtmodeltester, bookmarks,
web_history, quickmarks): web_history, quickmarks):
"""Test deleting a bookmark from the url completion model.""" """Test deleting a bookmark from the url completion model."""
config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'}
model = urlmodel.url() model = urlmodel.url()
model.set_pattern('') model.set_pattern('')
qtmodeltester.data_display_may_return_none = True qtmodeltester.data_display_may_return_none = True
@ -353,11 +352,10 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, bookmarks,
assert len_before == len(bookmarks.marks) + 1 assert len_before == len(bookmarks.marks) + 1
def test_url_completion_delete_quickmark(qtmodeltester, config_stub, def test_url_completion_delete_quickmark(qtmodeltester,
quickmarks, web_history, bookmarks, quickmarks, web_history, bookmarks,
qtbot): qtbot):
"""Test deleting a bookmark from the url completion model.""" """Test deleting a bookmark from the url completion model."""
config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'}
model = urlmodel.url() model = urlmodel.url()
model.set_pattern('') model.set_pattern('')
qtmodeltester.data_display_may_return_none = True qtmodeltester.data_display_may_return_none = True
@ -377,11 +375,10 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub,
assert len_before == len(quickmarks.marks) + 1 assert len_before == len(quickmarks.marks) + 1
def test_url_completion_delete_history(qtmodeltester, config_stub, def test_url_completion_delete_history(qtmodeltester,
web_history_populated, web_history_populated,
quickmarks, bookmarks): quickmarks, bookmarks):
"""Test deleting a history entry.""" """Test deleting a history entry."""
config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'}
model = urlmodel.url() model = urlmodel.url()
model.set_pattern('') model.set_pattern('')
qtmodeltester.data_display_may_return_none = True qtmodeltester.data_display_may_return_none = True
@ -598,14 +595,11 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub,
}) })
def test_url_completion_benchmark(benchmark, config_stub, def test_url_completion_benchmark(benchmark,
quickmark_manager_stub, quickmark_manager_stub,
bookmark_manager_stub, bookmark_manager_stub,
web_history): web_history):
"""Benchmark url completion.""" """Benchmark url completion."""
config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d',
'web-history-max-items': 1000}
r = range(100000) r = range(100000)
entries = { entries = {
'last_atime': list(r), 'last_atime': list(r),

View File

@ -1,158 +0,0 @@
# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) <ryan@rcorre.net>
#
# This file is part of qutebrowser.
#
# qutebrowser is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# qutebrowser is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
"""Test SQL-based completions."""
import unittest.mock
import pytest
from qutebrowser.misc import sql
from qutebrowser.completion.models import sqlcategory
from qutebrowser.commands import cmdexc
pytestmark = pytest.mark.usefixtures('init_sql')
@pytest.mark.parametrize('sort_by, sort_order, data, expected', [
(None, None,
[('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')],
[('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]),
('a', 'asc',
[('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')],
[('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]),
('a', 'desc',
[('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')],
[('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]),
('b', 'asc',
[('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')],
[('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]),
('b', 'desc',
[('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')],
[('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]),
('c', 'asc',
[('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)],
[('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]),
('c', 'desc',
[('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)],
[('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]),
])
def test_sorting(sort_by, sort_order, data, expected, model_validator):
table = sql.SqlTable('Foo', ['a', 'b', 'c'])
for row in data:
table.insert({'a': row[0], 'b': row[1], 'c': row[2]})
cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by,
sort_order=sort_order)
model_validator.set_model(cat)
cat.set_pattern('')
model_validator.validate(expected)
@pytest.mark.parametrize('pattern, filter_cols, before, after', [
('foo', [0],
[('foo', '', ''), ('bar', '', ''), ('aafobbb', '', '')],
[('foo',)]),
('foo', [0],
[('baz', 'bar', 'foo'), ('foo', '', ''), ('bar', 'foo', '')],
[('foo', '', '')]),
('foo', [0],
[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')],
[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]),
('foo', [1],
[('foo', 'bar', ''), ('bar', 'foo', '')],
[('bar', 'foo', '')]),
('foo', [0, 1],
[('foo', 'bar', ''), ('bar', 'foo', ''), ('biz', 'baz', 'foo')],
[('foo', 'bar', ''), ('bar', 'foo', '')]),
('foo', [0, 1, 2],
[('foo', '', ''), ('bar', '', ''), ('baz', 'bar', 'foo')],
[('foo', '', ''), ('baz', 'bar', 'foo')]),
('foo bar', [0],
[('foo', '', ''), ('bar foo', '', ''), ('xfooyybarz', '', '')],
[('xfooyybarz', '', '')]),
('foo%bar', [0],
[('foo%bar', '', ''), ('foo bar', '', ''), ('foobar', '', '')],
[('foo%bar', '', '')]),
('_', [0],
[('a_b', '', ''), ('__a', '', ''), ('abc', '', '')],
[('a_b', '', ''), ('__a', '', '')]),
('%', [0, 1],
[('\\foo', '\\bar', '')],
[]),
("can't", [0],
[("can't touch this", '', ''), ('a', '', '')],
[("can't touch this", '', '')]),
])
def test_set_pattern(pattern, filter_cols, before, after, model_validator):
"""Validate the filtering and sorting results of set_pattern."""
table = sql.SqlTable('Foo', ['a', 'b', 'c'])
for row in before:
table.insert({'a': row[0], 'b': row[1], 'c': row[2]})
filter_fields = [['a', 'b', 'c'][i] for i in filter_cols]
cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields)
model_validator.set_model(cat)
cat.set_pattern(pattern)
model_validator.validate(after)
def test_select(model_validator):
table = sql.SqlTable('Foo', ['a', 'b', 'c'])
table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'})
cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a')
model_validator.set_model(cat)
cat.set_pattern('')
model_validator.validate([('bar', 'baz', 'foo')])
def test_delete_cur_item():
table = sql.SqlTable('Foo', ['a', 'b'])
table.insert({'a': 'foo', 'b': 1})
table.insert({'a': 'bar', 'b': 2})
func = unittest.mock.Mock(spec=[])
cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], delete_func=func)
cat.set_pattern('')
cat.delete_cur_item(cat.index(0, 0))
func.assert_called_with(['foo', 1])
def test_delete_cur_item_no_func():
table = sql.SqlTable('Foo', ['a', 'b'])
table.insert({'a': 'foo', 'b': 1})
table.insert({'a': 'bar', 'b': 2})
cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'])
cat.set_pattern('')
with pytest.raises(cmdexc.CommandError, match='Cannot delete this item'):
cat.delete_cur_item(cat.index(0, 0))