From 08bb3f4f19b5117a00a7756119c4666d2065cfab Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 16 Sep 2016 15:41:54 -0400 Subject: [PATCH 001/161] Implement completion models as functions. First step of Completion Model/View revamping (#74). Rewrite the completion models as functions that each return an instance of a CompletionModel class. Caching is removed from all models except the UrlModel. Models other than the UrlModel can be generated very quickly so caching just adds needless complexity and can lead to incorrect results if one forgets to wire up a signal. --- qutebrowser/completion/completer.py | 17 +- qutebrowser/completion/completionwidget.py | 5 +- qutebrowser/completion/models/base.py | 25 +- qutebrowser/completion/models/configmodel.py | 179 +++------- qutebrowser/completion/models/instances.py | 193 ++--------- qutebrowser/completion/models/miscmodels.py | 293 +++++----------- qutebrowser/completion/models/sortfilter.py | 2 +- qutebrowser/completion/models/urlmodel.py | 324 +++++++++--------- tests/unit/completion/test_column_widths.py | 50 --- tests/unit/completion/test_completer.py | 104 +++--- .../unit/completion/test_completionwidget.py | 10 +- tests/unit/completion/test_models.py | 33 +- tests/unit/completion/test_sortfilter.py | 10 +- 13 files changed, 431 insertions(+), 814 deletions(-) delete mode 100644 tests/unit/completion/test_column_widths.py diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 96d937829..21af74da5 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -72,20 +72,7 @@ class Completer(QObject): Return: A completion model or None. """ - if completion == usertypes.Completion.option: - section = pos_args[0] - model = instances.get(completion).get(section) - elif completion == usertypes.Completion.value: - section = pos_args[0] - option = pos_args[1] - try: - model = instances.get(completion)[section][option] - except KeyError: - # No completion model for this section/option. - model = None - else: - model = instances.get(completion) - + model = instances.get(completion)(*pos_args) if model is None: return None else: @@ -109,7 +96,7 @@ class Completer(QObject): log.completion.debug("After removing flags: {}".format(before_cursor)) if not before_cursor: # '|' or 'set|' - model = instances.get(usertypes.Completion.command) + model = instances.get(usertypes.Completion.command)() return sortfilter.CompletionFilterModel(source=model, parent=self) try: cmd = cmdutils.cmd_dict[before_cursor[0]] diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 490fcd6c0..f12027340 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -28,7 +28,6 @@ from PyQt5.QtCore import pyqtSlot, pyqtSignal, Qt, QItemSelectionModel, QSize from qutebrowser.config import config, style from qutebrowser.completion import completiondelegate -from qutebrowser.completion.models import base from qutebrowser.utils import utils, usertypes, objreg from qutebrowser.commands import cmdexc, cmdutils @@ -112,7 +111,7 @@ class CompletionView(QTreeView): # objreg.get('config').changed.connect(self.init_command_completion) objreg.get('config').changed.connect(self._on_config_changed) - self._column_widths = base.BaseCompletionModel.COLUMN_WIDTHS + self._column_widths = (30, 70, 0) self._active = False self._delegate = completiondelegate.CompletionItemDelegate(self) @@ -300,7 +299,7 @@ class CompletionView(QTreeView): if pattern is not None: model.set_pattern(pattern) - self._column_widths = model.srcmodel.COLUMN_WIDTHS + self._column_widths = model.srcmodel.column_widths self._resize_columns() self._maybe_update_geometry() diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py index b1cad276a..43f3a1b48 100644 --- a/qutebrowser/completion/models/base.py +++ b/qutebrowser/completion/models/base.py @@ -33,26 +33,27 @@ Role = usertypes.enum('Role', ['sort', 'userdata'], start=Qt.UserRole, is_int=True) -class BaseCompletionModel(QStandardItemModel): +class CompletionModel(QStandardItemModel): """A simple QStandardItemModel adopted for completions. Used for showing completions later in the CompletionView. Supports setting marks and adding new categories/items easily. - Class Attributes: - COLUMN_WIDTHS: The width percentages of the columns used in the - completion view. - DUMB_SORT: the dumb sorting used by the model + Attributes: + column_widths: The width percentages of the columns used in the + completion view. + dumb_sort: the dumb sorting used by the model """ - COLUMN_WIDTHS = (30, 70, 0) - DUMB_SORT = None - - def __init__(self, parent=None): + def __init__(self, dumb_sort=None, column_widths=(30, 70, 0), + columns_to_filter=None, delete_cur_item=None, parent=None): super().__init__(parent) self.setColumnCount(3) - self.columns_to_filter = [0] + self.columns_to_filter = columns_to_filter or [0] + self.dumb_sort = dumb_sort + self.column_widths = column_widths + self.delete_cur_item = delete_cur_item def new_category(self, name, sort=None): """Add a new category to the model. @@ -103,10 +104,6 @@ class BaseCompletionModel(QStandardItemModel): nameitem.setData(userdata, Role.userdata) return nameitem, descitem, miscitem - def delete_cur_item(self, completion): - """Delete the selected item.""" - raise NotImplementedError - def flags(self, index): """Return the item flags for index. diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index c9e9850d1..5ed2a47d0 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -17,142 +17,75 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""CompletionModels for the config.""" +"""Functions that return config-related completion models.""" -from PyQt5.QtCore import pyqtSlot, Qt - -from qutebrowser.config import config, configdata -from qutebrowser.utils import log, qtutils, objreg +from qutebrowser.config import config, configdata, configexc from qutebrowser.completion.models import base -class SettingSectionCompletionModel(base.BaseCompletionModel): - +def section(): """A CompletionModel filled with settings sections.""" - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - COLUMN_WIDTHS = (20, 70, 10) - - def __init__(self, parent=None): - super().__init__(parent) - cat = self.new_category("Sections") - for name in configdata.DATA: - desc = configdata.SECTION_DESC[name].splitlines()[0].strip() - self.new_item(cat, name, desc) + model = base.CompletionModel(column_widths=(20, 70, 10)) + cat = model.new_category("Sections") + for name in configdata.DATA: + desc = configdata.SECTION_DESC[name].splitlines()[0].strip() + model.new_item(cat, name, desc) + return model -class SettingOptionCompletionModel(base.BaseCompletionModel): - +def option(sectname): """A CompletionModel filled with settings and their descriptions. - Attributes: - _misc_items: A dict of the misc. column items which will be set later. - _section: The config section this model shows. + Args: + sectname: The name of the config section this model shows. """ - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - COLUMN_WIDTHS = (20, 70, 10) - - def __init__(self, section, parent=None): - super().__init__(parent) - cat = self.new_category(section) - sectdata = configdata.DATA[section] - self._misc_items = {} - self._section = section - objreg.get('config').changed.connect(self.update_misc_column) - for name in sectdata: - try: - desc = sectdata.descriptions[name] - except (KeyError, AttributeError): - # Some stuff (especially ValueList items) don't have a - # description. - desc = "" - else: - desc = desc.splitlines()[0] - value = config.get(section, name, raw=True) - _valitem, _descitem, miscitem = self.new_item(cat, name, desc, - value) - self._misc_items[name] = miscitem - - @pyqtSlot(str, str) - def update_misc_column(self, section, option): - """Update misc column when config changed.""" - if section != self._section: - return + model = base.CompletionModel(column_widths=(20, 70, 10)) + cat = model.new_category(sectname) + try: + sectdata = configdata.DATA[sectname] + except KeyError: + return None + for name in sectdata: try: - item = self._misc_items[option] - except KeyError: - log.completion.debug("Couldn't get item {}.{} from model!".format( - section, option)) - # changed before init - return - val = config.get(section, option, raw=True) - idx = item.index() - qtutils.ensure_valid(idx) - ok = self.setData(idx, val, Qt.DisplayRole) - if not ok: - raise ValueError("Setting data failed! (section: {}, option: {}, " - "value: {})".format(section, option, val)) + desc = sectdata.descriptions[name] + except (KeyError, AttributeError): + # Some stuff (especially ValueList items) don't have a + # description. + desc = "" + else: + desc = desc.splitlines()[0] + val = config.get(sectname, name, raw=True) + model.new_item(cat, name, desc, val) + return model -class SettingValueCompletionModel(base.BaseCompletionModel): - +def value(sectname, optname): """A CompletionModel filled with setting values. - Attributes: - _section: The config section this model shows. - _option: The config option this model shows. + Args: + sectname: The name of the config section this model shows. + optname: The name of the config option this model shows. """ - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - COLUMN_WIDTHS = (20, 70, 10) - - def __init__(self, section, option, parent=None): - super().__init__(parent) - self._section = section - self._option = option - objreg.get('config').changed.connect(self.update_current_value) - cur_cat = self.new_category("Current/Default", sort=0) - value = config.get(section, option, raw=True) - if not value: - value = '""' - self.cur_item, _descitem, _miscitem = self.new_item(cur_cat, value, - "Current value") - default_value = configdata.DATA[section][option].default() - if not default_value: - default_value = '""' - self.new_item(cur_cat, default_value, "Default value") - if hasattr(configdata.DATA[section], 'valtype'): - # Same type for all values (ValueList) - vals = configdata.DATA[section].valtype.complete() - else: - if option is None: - raise ValueError("option may only be None for ValueList " - "sections, but {} is not!".format(section)) - # Different type for each value (KeyValue) - vals = configdata.DATA[section][option].typ.complete() - if vals is not None: - cat = self.new_category("Completions", sort=1) - for (val, desc) in vals: - self.new_item(cat, val, desc) - - @pyqtSlot(str, str) - def update_current_value(self, section, option): - """Update current value when config changed.""" - if (section, option) != (self._section, self._option): - return - value = config.get(section, option, raw=True) - if not value: - value = '""' - idx = self.cur_item.index() - qtutils.ensure_valid(idx) - ok = self.setData(idx, value, Qt.DisplayRole) - if not ok: - raise ValueError("Setting data failed! (section: {}, option: {}, " - "value: {})".format(section, option, value)) + model = base.CompletionModel(column_widths=(20, 70, 10)) + cur_cat = model.new_category("Current/Default", sort=0) + try: + val = config.get(sectname, optname, raw=True) or '""' + except (configexc.NoSectionError, configexc.NoOptionError): + return None + model.new_item(cur_cat, val, "Current value") + default_value = configdata.DATA[sectname][optname].default() or '""' + model.new_item(cur_cat, default_value, "Default value") + if hasattr(configdata.DATA[sectname], 'valtype'): + # Same type for all values (ValueList) + vals = configdata.DATA[sectname].valtype.complete() + else: + if optname is None: + raise ValueError("optname may only be None for ValueList " + "sections, but {} is not!".format(sectname)) + # Different type for each value (KeyValue) + vals = configdata.DATA[sectname][optname].typ.complete() + if vals is not None: + cat = model.new_category("Completions", sort=1) + for (val, desc) in vals: + model.new_item(cat, val, desc) + return model diff --git a/qutebrowser/completion/models/instances.py b/qutebrowser/completion/models/instances.py index f7eaaca86..ce109ae7a 100644 --- a/qutebrowser/completion/models/instances.py +++ b/qutebrowser/completion/models/instances.py @@ -17,180 +17,37 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""Global instances of the completion models. - -Module attributes: - _instances: A dict of available completions. - INITIALIZERS: A {usertypes.Completion: callable} dict of functions to - initialize completions. -""" - -import functools +"""Global instances of the completion models.""" +from qutebrowser.utils import usertypes from qutebrowser.completion.models import miscmodels, urlmodel, configmodel -from qutebrowser.utils import objreg, usertypes, log, debug -from qutebrowser.config import configdata, config - - -_instances = {} - - -def _init_command_completion(): - """Initialize the command completion model.""" - log.completion.debug("Initializing command completion.") - model = miscmodels.CommandCompletionModel() - _instances[usertypes.Completion.command] = model - - -def _init_helptopic_completion(): - """Initialize the helptopic completion model.""" - log.completion.debug("Initializing helptopic completion.") - model = miscmodels.HelpCompletionModel() - _instances[usertypes.Completion.helptopic] = model - - -def _init_url_completion(): - """Initialize the URL completion model.""" - log.completion.debug("Initializing URL completion.") - with debug.log_time(log.completion, 'URL completion init'): - model = urlmodel.UrlCompletionModel() - _instances[usertypes.Completion.url] = model - - -def _init_tab_completion(): - """Initialize the tab completion model.""" - log.completion.debug("Initializing tab completion.") - with debug.log_time(log.completion, 'tab completion init'): - model = miscmodels.TabCompletionModel() - _instances[usertypes.Completion.tab] = model - - -def _init_setting_completions(): - """Initialize setting completion models.""" - log.completion.debug("Initializing setting completion.") - _instances[usertypes.Completion.section] = ( - configmodel.SettingSectionCompletionModel()) - _instances[usertypes.Completion.option] = {} - _instances[usertypes.Completion.value] = {} - for sectname in configdata.DATA: - opt_model = configmodel.SettingOptionCompletionModel(sectname) - _instances[usertypes.Completion.option][sectname] = opt_model - _instances[usertypes.Completion.value][sectname] = {} - for opt in configdata.DATA[sectname]: - val_model = configmodel.SettingValueCompletionModel(sectname, opt) - _instances[usertypes.Completion.value][sectname][opt] = val_model - - -def init_quickmark_completions(): - """Initialize quickmark completion models.""" - log.completion.debug("Initializing quickmark completion.") - try: - _instances[usertypes.Completion.quickmark_by_name].deleteLater() - except KeyError: - pass - model = miscmodels.QuickmarkCompletionModel() - _instances[usertypes.Completion.quickmark_by_name] = model - - -def init_bookmark_completions(): - """Initialize bookmark completion models.""" - log.completion.debug("Initializing bookmark completion.") - try: - _instances[usertypes.Completion.bookmark_by_url].deleteLater() - except KeyError: - pass - model = miscmodels.BookmarkCompletionModel() - _instances[usertypes.Completion.bookmark_by_url] = model - - -def init_session_completion(): - """Initialize session completion model.""" - log.completion.debug("Initializing session completion.") - try: - _instances[usertypes.Completion.sessions].deleteLater() - except KeyError: - pass - model = miscmodels.SessionCompletionModel() - _instances[usertypes.Completion.sessions] = model - - -def _init_bind_completion(): - """Initialize the command completion model.""" - log.completion.debug("Initializing bind completion.") - model = miscmodels.BindCompletionModel() - _instances[usertypes.Completion.bind] = model - - -INITIALIZERS = { - usertypes.Completion.command: _init_command_completion, - usertypes.Completion.helptopic: _init_helptopic_completion, - usertypes.Completion.url: _init_url_completion, - usertypes.Completion.tab: _init_tab_completion, - usertypes.Completion.section: _init_setting_completions, - usertypes.Completion.option: _init_setting_completions, - usertypes.Completion.value: _init_setting_completions, - usertypes.Completion.quickmark_by_name: init_quickmark_completions, - usertypes.Completion.bookmark_by_url: init_bookmark_completions, - usertypes.Completion.sessions: init_session_completion, - usertypes.Completion.bind: _init_bind_completion, -} def get(completion): """Get a certain completion. Initializes the completion if needed.""" - try: - return _instances[completion] - except KeyError: - if completion in INITIALIZERS: - INITIALIZERS[completion]() - return _instances[completion] - else: - raise - - -def update(completions): - """Update an already existing completion. - - Args: - completions: An iterable of usertypes.Completions. - """ - did_run = [] - for completion in completions: - if completion in _instances: - func = INITIALIZERS[completion] - if func not in did_run: - func() - did_run.append(func) - - -@config.change_filter('aliases', function=True) -def _update_aliases(): - """Update completions that include command aliases.""" - update([usertypes.Completion.command]) + if completion == usertypes.Completion.command: + return miscmodels.command + if completion == usertypes.Completion.helptopic: + return miscmodels.helptopic + if completion == usertypes.Completion.tab: + return miscmodels.buffer + if completion == usertypes.Completion.quickmark_by_name: + return miscmodels.quickmark + if completion == usertypes.Completion.bookmark_by_url: + return miscmodels.bookmark + if completion == usertypes.Completion.sessions: + return miscmodels.session + if completion == usertypes.Completion.bind: + return miscmodels.bind + if completion == usertypes.Completion.section: + return configmodel.section + if completion == usertypes.Completion.option: + return configmodel.option + if completion == usertypes.Completion.value: + return configmodel.value + if completion == usertypes.Completion.url: + return urlmodel.url def init(): - """Initialize completions. Note this only connects signals.""" - quickmark_manager = objreg.get('quickmark-manager') - quickmark_manager.changed.connect( - functools.partial(update, [usertypes.Completion.quickmark_by_name])) - - bookmark_manager = objreg.get('bookmark-manager') - bookmark_manager.changed.connect( - functools.partial(update, [usertypes.Completion.bookmark_by_url])) - - session_manager = objreg.get('session-manager') - session_manager.update_completion.connect( - functools.partial(update, [usertypes.Completion.sessions])) - - history = objreg.get('web-history') - history.async_read_done.connect( - functools.partial(update, [usertypes.Completion.url])) - - keyconf = objreg.get('key-config') - keyconf.changed.connect( - functools.partial(update, [usertypes.Completion.command])) - keyconf.changed.connect( - functools.partial(update, [usertypes.Completion.bind])) - - objreg.get('config').changed.connect(_update_aliases) + pass diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 5ab381c43..bcbb94177 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -17,252 +17,139 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""Misc. CompletionModels.""" +"""Functions that return miscellaneous completion models.""" -from PyQt5.QtCore import Qt, QTimer, pyqtSlot +from PyQt5.QtCore import Qt -from qutebrowser.browser import browsertab from qutebrowser.config import config, configdata from qutebrowser.utils import objreg, log, qtutils from qutebrowser.commands import cmdutils from qutebrowser.completion.models import base -class CommandCompletionModel(base.BaseCompletionModel): - +def command(): """A CompletionModel filled with non-hidden commands and descriptions.""" - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - COLUMN_WIDTHS = (20, 60, 20) - - def __init__(self, parent=None): - super().__init__(parent) - cmdlist = _get_cmd_completions(include_aliases=True, - include_hidden=False) - cat = self.new_category("Commands") - for (name, desc, misc) in cmdlist: - self.new_item(cat, name, desc, misc) + model = base.CompletionModel(column_widths=(20, 60, 20)) + cmdlist = _get_cmd_completions(include_aliases=True, include_hidden=False) + cat = model.new_category("Commands") + for (name, desc, misc) in cmdlist: + model.new_item(cat, name, desc, misc) + return model -class HelpCompletionModel(base.BaseCompletionModel): - +def helptopic(): """A CompletionModel filled with help topics.""" + model = base.CompletionModel() - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method + cmdlist = _get_cmd_completions(include_aliases=False, include_hidden=True, + prefix=':') + cat = model.new_category("Commands") + for (name, desc, misc) in cmdlist: + model.new_item(cat, name, desc, misc) - COLUMN_WIDTHS = (20, 60, 20) - - def __init__(self, parent=None): - super().__init__(parent) - self._init_commands() - self._init_settings() - - def _init_commands(self): - """Fill completion with :command entries.""" - cmdlist = _get_cmd_completions(include_aliases=False, - include_hidden=True, prefix=':') - cat = self.new_category("Commands") - for (name, desc, misc) in cmdlist: - self.new_item(cat, name, desc, misc) - - def _init_settings(self): - """Fill completion with section->option entries.""" - cat = self.new_category("Settings") - for sectname, sectdata in configdata.DATA.items(): - for optname in sectdata: - try: - desc = sectdata.descriptions[optname] - except (KeyError, AttributeError): - # Some stuff (especially ValueList items) don't have a - # description. - desc = "" - else: - desc = desc.splitlines()[0] - name = '{}->{}'.format(sectname, optname) - self.new_item(cat, name, desc) + cat = model.new_category("Settings") + for sectname, sectdata in configdata.DATA.items(): + for optname in sectdata: + try: + desc = sectdata.descriptions[optname] + except (KeyError, AttributeError): + # Some stuff (especially ValueList items) don't have a + # description. + desc = "" + else: + desc = desc.splitlines()[0] + name = '{}->{}'.format(sectname, optname) + model.new_item(cat, name, desc) + return model -class QuickmarkCompletionModel(base.BaseCompletionModel): - +def quickmark(): """A CompletionModel filled with all quickmarks.""" - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - def __init__(self, parent=None): - super().__init__(parent) - cat = self.new_category("Quickmarks") - quickmarks = objreg.get('quickmark-manager').marks.items() - for qm_name, qm_url in quickmarks: - self.new_item(cat, qm_name, qm_url) + model = base.CompletionModel() + cat = model.new_category("Quickmarks") + quickmarks = objreg.get('quickmark-manager').marks.items() + for qm_name, qm_url in quickmarks: + model.new_item(cat, qm_name, qm_url) + return model -class BookmarkCompletionModel(base.BaseCompletionModel): - +def bookmark(): """A CompletionModel filled with all bookmarks.""" - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - def __init__(self, parent=None): - super().__init__(parent) - cat = self.new_category("Bookmarks") - bookmarks = objreg.get('bookmark-manager').marks.items() - for bm_url, bm_title in bookmarks: - self.new_item(cat, bm_url, bm_title) + model = base.CompletionModel() + cat = model.new_category("Bookmarks") + bookmarks = objreg.get('bookmark-manager').marks.items() + for bm_url, bm_title in bookmarks: + model.new_item(cat, bm_url, bm_title) + return model -class SessionCompletionModel(base.BaseCompletionModel): - +def session(): """A CompletionModel filled with session names.""" - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - def __init__(self, parent=None): - super().__init__(parent) - cat = self.new_category("Sessions") - try: - for name in objreg.get('session-manager').list_sessions(): - if not name.startswith('_'): - self.new_item(cat, name) - except OSError: - log.completion.exception("Failed to list sessions!") + model = base.CompletionModel() + cat = model.new_category("Sessions") + try: + for name in objreg.get('session-manager').list_sessions(): + if not name.startswith('_'): + model.new_item(cat, name) + except OSError: + log.completion.exception("Failed to list sessions!") + return model -class TabCompletionModel(base.BaseCompletionModel): - +def buffer(): """A model to complete on open tabs across all windows. Used for switching the buffer command. """ + idx_column = 0 + url_column = 1 + text_column = 2 - IDX_COLUMN = 0 - URL_COLUMN = 1 - TEXT_COLUMN = 2 - - COLUMN_WIDTHS = (6, 40, 54) - DUMB_SORT = Qt.DescendingOrder - - def __init__(self, parent=None): - super().__init__(parent) - - self.columns_to_filter = [self.IDX_COLUMN, self.URL_COLUMN, - self.TEXT_COLUMN] - - for win_id in objreg.window_registry: - tabbed_browser = objreg.get('tabbed-browser', scope='window', - window=win_id) - for i in range(tabbed_browser.count()): - tab = tabbed_browser.widget(i) - tab.url_changed.connect(self.rebuild) - tab.title_changed.connect(self.rebuild) - tab.shutting_down.connect(self.delayed_rebuild) - tabbed_browser.new_tab.connect(self.on_new_tab) - tabbed_browser.tabBar().tabMoved.connect(self.rebuild) - objreg.get("app").new_window.connect(self.on_new_window) - self.rebuild() - - def on_new_window(self, window): - """Add hooks to new windows.""" - window.tabbed_browser.new_tab.connect(self.on_new_tab) - - @pyqtSlot(browsertab.AbstractTab) - def on_new_tab(self, tab): - """Add hooks to new tabs.""" - tab.url_changed.connect(self.rebuild) - tab.title_changed.connect(self.rebuild) - tab.shutting_down.connect(self.delayed_rebuild) - self.rebuild() - - @pyqtSlot() - def delayed_rebuild(self): - """Fire a rebuild indirectly so widgets get a chance to update.""" - QTimer.singleShot(0, self.rebuild) - - @pyqtSlot() - def rebuild(self): - """Rebuild completion model from current tabs. - - Very lazy method of keeping the model up to date. We could connect to - signals for new tab, tab url/title changed, tab close, tab moved and - make sure we handled background loads too ... but iterating over a - few/few dozen/few hundred tabs doesn't take very long at all. - """ - window_count = 0 - for win_id in objreg.window_registry: - tabbed_browser = objreg.get('tabbed-browser', scope='window', - window=win_id) - if not tabbed_browser.shutting_down: - window_count += 1 - - if window_count < self.rowCount(): - self.removeRows(window_count, self.rowCount() - window_count) - - for i, win_id in enumerate(objreg.window_registry): - tabbed_browser = objreg.get('tabbed-browser', scope='window', - window=win_id) - if tabbed_browser.shutting_down: - continue - if i >= self.rowCount(): - c = self.new_category("{}".format(win_id)) - else: - c = self.item(i, 0) - c.setData("{}".format(win_id), Qt.DisplayRole) - if tabbed_browser.count() < c.rowCount(): - c.removeRows(tabbed_browser.count(), - c.rowCount() - tabbed_browser.count()) - for idx in range(tabbed_browser.count()): - tab = tabbed_browser.widget(idx) - if idx >= c.rowCount(): - self.new_item(c, "{}/{}".format(win_id, idx + 1), - tab.url().toDisplayString(), - tabbed_browser.page_title(idx)) - else: - c.child(idx, 0).setData("{}/{}".format(win_id, idx + 1), - Qt.DisplayRole) - c.child(idx, 1).setData(tab.url().toDisplayString(), - Qt.DisplayRole) - c.child(idx, 2).setData(tabbed_browser.page_title(idx), - Qt.DisplayRole) - - def delete_cur_item(self, completion): - """Delete the selected item. - - Args: - completion: The Completion object to use. - """ + def delete_buffer(completion): + """Close the selected tab.""" index = completion.currentIndex() qtutils.ensure_valid(index) category = index.parent() qtutils.ensure_valid(category) - index = category.child(index.row(), self.IDX_COLUMN) + index = category.child(index.row(), idx_column) win_id, tab_index = index.data().split('/') - tabbed_browser = objreg.get('tabbed-browser', scope='window', window=int(win_id)) tabbed_browser.on_tab_close_requested(int(tab_index) - 1) + model = base.CompletionModel( + column_widths=(6, 40, 54), + dumb_sort=Qt.DescendingOrder, + delete_cur_item=delete_buffer, + columns_to_filter=[idx_column, url_column, text_column]) -class BindCompletionModel(base.BaseCompletionModel): + for win_id in objreg.window_registry: + tabbed_browser = objreg.get('tabbed-browser', scope='window', + window=win_id) + if tabbed_browser.shutting_down: + continue + c = model.new_category("{}".format(win_id)) + for idx in range(tabbed_browser.count()): + tab = tabbed_browser.widget(idx) + model.new_item(c, "{}/{}".format(win_id, idx + 1), + tab.url().toDisplayString(), + tabbed_browser.page_title(idx)) + return model - """A CompletionModel filled with all bindable commands and descriptions.""" - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method +def bind(_): + """A CompletionModel filled with all bindable commands and descriptions. - COLUMN_WIDTHS = (20, 60, 20) - - def __init__(self, parent=None): - super().__init__(parent) - cmdlist = _get_cmd_completions(include_hidden=True, - include_aliases=True) - cat = self.new_category("Commands") - for (name, desc, misc) in cmdlist: - self.new_item(cat, name, desc, misc) + Args: + _: the key being bound. + """ + # TODO: offer a 'Current binding' completion based on the key. + model = base.CompletionModel(column_widths=(20, 60, 20)) + cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) + cat = model.new_category("Commands") + for (name, desc, misc) in cmdlist: + model.new_item(cat, name, desc, misc) + return model def _get_cmd_completions(include_hidden, include_aliases, prefix=''): diff --git a/qutebrowser/completion/models/sortfilter.py b/qutebrowser/completion/models/sortfilter.py index e2db88b9e..92dd1b2a0 100644 --- a/qutebrowser/completion/models/sortfilter.py +++ b/qutebrowser/completion/models/sortfilter.py @@ -50,7 +50,7 @@ class CompletionFilterModel(QSortFilterProxyModel): self.pattern = '' self.pattern_re = None - dumb_sort = self.srcmodel.DUMB_SORT + dumb_sort = self.srcmodel.dumb_sort if dumb_sort is None: # pylint: disable=invalid-name self.lessThan = self.intelligentLessThan diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 98f68c08c..d8b7f42a2 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""CompletionModels for URLs.""" +"""Function to return the url completion model for the `open` command.""" import datetime @@ -27,166 +27,172 @@ from qutebrowser.utils import objreg, utils, qtutils, log from qutebrowser.completion.models import base from qutebrowser.config import config +_URL_COLUMN = 0 +_TEXT_COLUMN = 1 +_TIME_COLUMN = 2 +_model = None +_history_cat = None +_quickmark_cat = None +_bookmark_cat = None -class UrlCompletionModel(base.BaseCompletionModel): - """A model which combines bookmarks, quickmarks and web history URLs. +def _delete_url(completion): + index = completion.currentIndex() + qtutils.ensure_valid(index) + category = index.parent() + index = category.child(index.row(), _URL_COLUMN) + qtutils.ensure_valid(category) + if category.data() == 'Bookmarks': + bookmark_manager = objreg.get('bookmark-manager') + bookmark_manager.delete(index.data()) + elif category.data() == 'Quickmarks': + quickmark_manager = objreg.get('quickmark-manager') + sibling = index.sibling(index.row(), _TEXT_COLUMN) + qtutils.ensure_valid(sibling) + name = sibling.data() + quickmark_manager.delete(name) + + +def _remove_oldest_history(): + """Remove the oldest history entry.""" + _history_cat.removeRow(0) + + +def _add_history_entry(entry): + """Add a new history entry to the completion.""" + _model.new_item(_history_cat, entry.url.toDisplayString(), + entry.title, _fmt_atime(entry.atime), + sort=int(entry.atime), userdata=entry.url) + + max_history = config.get('completion', 'web-history-max-items') + if max_history != -1 and _history_cat.rowCount() > max_history: + _remove_oldest_history() + + +@config.change_filter('completion', 'timestamp-format') +def _reformat_timestamps(): + """Reformat the timestamps if the config option was changed.""" + for i in range(_history_cat.rowCount()): + url_item = _history_cat.child(i, _URL_COLUMN) + atime_item = _history_cat.child(i, _TIME_COLUMN) + atime = url_item.data(base.Role.sort) + atime_item.setText(_fmt_atime(atime)) + + +@pyqtSlot(object) +def _on_history_item_added(entry): + """Slot called when a new history item was added.""" + for i in range(_history_cat.rowCount()): + url_item = _history_cat.child(i, _URL_COLUMN) + atime_item = _history_cat.child(i, _TIME_COLUMN) + title_item = _history_cat.child(i, _TEXT_COLUMN) + if url_item.data(base.Role.userdata) == entry.url: + atime_item.setText(_fmt_atime(entry.atime)) + title_item.setText(entry.title) + url_item.setData(int(entry.atime), base.Role.sort) + break + else: + _add_history_entry(entry) + + +@pyqtSlot() +def _on_history_cleared(): + _history_cat.removeRows(0, _history_cat.rowCount()) + + +def _remove_item(data, category, column): + """Helper function for on_quickmark_removed and on_bookmark_removed. + + Args: + data: The item to search for. + category: The category to search in. + column: The column to use for matching. + """ + for i in range(category.rowCount()): + item = category.child(i, column) + if item.data(Qt.DisplayRole) == data: + category.removeRow(i) + break + + +@pyqtSlot(str) +def _on_quickmark_removed(name): + """Called when a quickmark has been removed by the user. + + Args: + name: The name of the quickmark which has been removed. + """ + _remove_item(name, _quickmark_cat, _TEXT_COLUMN) + + +@pyqtSlot(str) +def _on_bookmark_removed(urltext): + """Called when a bookmark has been removed by the user. + + Args: + urltext: The url of the bookmark which has been removed. + """ + _remove_item(urltext, _bookmark_cat, _URL_COLUMN) + + +def _fmt_atime(atime): + """Format an atime to a human-readable string.""" + fmt = config.get('completion', 'timestamp-format') + if fmt is None: + return '' + try: + dt = datetime.datetime.fromtimestamp(atime) + except (ValueError, OSError, OverflowError): + # Different errors which can occur for too large values... + log.misc.error("Got invalid timestamp {}!".format(atime)) + return '(invalid)' + else: + return dt.strftime(fmt) + + +def _init(): + global _model, _quickmark_cat, _bookmark_cat, _history_cat + _model = base.CompletionModel(column_widths=(40, 50, 10), + dumb_sort=Qt.DescendingOrder, + delete_cur_item=_delete_url, + columns_to_filter=[_URL_COLUMN, + _TEXT_COLUMN]) + _quickmark_cat = _model.new_category("Quickmarks") + _bookmark_cat = _model.new_category("Bookmarks") + _history_cat = _model.new_category("History") + + quickmark_manager = objreg.get('quickmark-manager') + quickmarks = quickmark_manager.marks.items() + for qm_name, qm_url in quickmarks: + _model.new_item(_quickmark_cat, qm_url, qm_name) + quickmark_manager.added.connect( + lambda name, url: _model.new_item(_quickmark_cat, url, name)) + quickmark_manager.removed.connect(_on_quickmark_removed) + + bookmark_manager = objreg.get('bookmark-manager') + bookmarks = bookmark_manager.marks.items() + for bm_url, bm_title in bookmarks: + _model.new_item(_bookmark_cat, bm_url, bm_title) + bookmark_manager.added.connect( + lambda name, url: _model.new_item(_bookmark_cat, url, name)) + bookmark_manager.removed.connect(_on_bookmark_removed) + + history = objreg.get('web-history') + max_history = config.get('completion', 'web-history-max-items') + for entry in utils.newest_slice(history, max_history): + if not entry.redirect: + _add_history_entry(entry) + history.add_completion_item.connect(_on_history_item_added) + history.cleared.connect(_on_history_cleared) + + objreg.get('config').changed.connect(_reformat_timestamps) + + +def url(): + """A _model which combines bookmarks, quickmarks and web history URLs. Used for the `open` command. """ - - URL_COLUMN = 0 - TEXT_COLUMN = 1 - TIME_COLUMN = 2 - - COLUMN_WIDTHS = (40, 50, 10) - DUMB_SORT = Qt.DescendingOrder - - def __init__(self, parent=None): - super().__init__(parent) - - self.columns_to_filter = [self.URL_COLUMN, self.TEXT_COLUMN] - - self._quickmark_cat = self.new_category("Quickmarks") - self._bookmark_cat = self.new_category("Bookmarks") - self._history_cat = self.new_category("History") - - quickmark_manager = objreg.get('quickmark-manager') - quickmarks = quickmark_manager.marks.items() - for qm_name, qm_url in quickmarks: - self.new_item(self._quickmark_cat, qm_url, qm_name) - quickmark_manager.added.connect( - lambda name, url: self.new_item(self._quickmark_cat, url, name)) - quickmark_manager.removed.connect(self.on_quickmark_removed) - - bookmark_manager = objreg.get('bookmark-manager') - bookmarks = bookmark_manager.marks.items() - for bm_url, bm_title in bookmarks: - self.new_item(self._bookmark_cat, bm_url, bm_title) - bookmark_manager.added.connect( - lambda name, url: self.new_item(self._bookmark_cat, url, name)) - bookmark_manager.removed.connect(self.on_bookmark_removed) - - self._history = objreg.get('web-history') - self._max_history = config.get('completion', 'web-history-max-items') - history = utils.newest_slice(self._history, self._max_history) - for entry in history: - if not entry.redirect: - self._add_history_entry(entry) - self._history.add_completion_item.connect(self.on_history_item_added) - self._history.cleared.connect(self.on_history_cleared) - - objreg.get('config').changed.connect(self.reformat_timestamps) - - def _fmt_atime(self, atime): - """Format an atime to a human-readable string.""" - fmt = config.get('completion', 'timestamp-format') - if fmt is None: - return '' - try: - dt = datetime.datetime.fromtimestamp(atime) - except (ValueError, OSError, OverflowError): - # Different errors which can occur for too large values... - log.misc.error("Got invalid timestamp {}!".format(atime)) - return '(invalid)' - else: - return dt.strftime(fmt) - - def _remove_oldest_history(self): - """Remove the oldest history entry.""" - self._history_cat.removeRow(0) - - def _add_history_entry(self, entry): - """Add a new history entry to the completion.""" - self.new_item(self._history_cat, entry.url.toDisplayString(), - entry.title, - self._fmt_atime(entry.atime), sort=int(entry.atime), - userdata=entry.url) - - if (self._max_history != -1 and - self._history_cat.rowCount() > self._max_history): - self._remove_oldest_history() - - @config.change_filter('completion', 'timestamp-format') - def reformat_timestamps(self): - """Reformat the timestamps if the config option was changed.""" - for i in range(self._history_cat.rowCount()): - url_item = self._history_cat.child(i, self.URL_COLUMN) - atime_item = self._history_cat.child(i, self.TIME_COLUMN) - atime = url_item.data(base.Role.sort) - atime_item.setText(self._fmt_atime(atime)) - - @pyqtSlot(object) - def on_history_item_added(self, entry): - """Slot called when a new history item was added.""" - for i in range(self._history_cat.rowCount()): - url_item = self._history_cat.child(i, self.URL_COLUMN) - atime_item = self._history_cat.child(i, self.TIME_COLUMN) - title_item = self._history_cat.child(i, self.TEXT_COLUMN) - url = url_item.data(base.Role.userdata) - if url == entry.url: - atime_item.setText(self._fmt_atime(entry.atime)) - title_item.setText(entry.title) - url_item.setData(int(entry.atime), base.Role.sort) - break - else: - self._add_history_entry(entry) - - @pyqtSlot() - def on_history_cleared(self): - self._history_cat.removeRows(0, self._history_cat.rowCount()) - - def _remove_item(self, data, category, column): - """Helper function for on_quickmark_removed and on_bookmark_removed. - - Args: - data: The item to search for. - category: The category to search in. - column: The column to use for matching. - """ - for i in range(category.rowCount()): - item = category.child(i, column) - if item.data(Qt.DisplayRole) == data: - category.removeRow(i) - break - - @pyqtSlot(str) - def on_quickmark_removed(self, name): - """Called when a quickmark has been removed by the user. - - Args: - name: The name of the quickmark which has been removed. - """ - self._remove_item(name, self._quickmark_cat, self.TEXT_COLUMN) - - @pyqtSlot(str) - def on_bookmark_removed(self, url): - """Called when a bookmark has been removed by the user. - - Args: - url: The url of the bookmark which has been removed. - """ - self._remove_item(url, self._bookmark_cat, self.URL_COLUMN) - - def delete_cur_item(self, completion): - """Delete the selected item. - - Args: - completion: The Completion object to use. - """ - index = completion.currentIndex() - qtutils.ensure_valid(index) - category = index.parent() - index = category.child(index.row(), self.URL_COLUMN) - url = index.data() - qtutils.ensure_valid(category) - - if category.data() == 'Bookmarks': - bookmark_manager = objreg.get('bookmark-manager') - bookmark_manager.delete(url) - elif category.data() == 'Quickmarks': - quickmark_manager = objreg.get('quickmark-manager') - sibling = index.sibling(index.row(), self.TEXT_COLUMN) - qtutils.ensure_valid(sibling) - name = sibling.data() - quickmark_manager.delete(name) + if not _model: + _init() + return _model diff --git a/tests/unit/completion/test_column_widths.py b/tests/unit/completion/test_column_widths.py deleted file mode 100644 index 21456ed37..000000000 --- a/tests/unit/completion/test_column_widths.py +++ /dev/null @@ -1,50 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2015-2017 Alexander Cogneau -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Tests for qutebrowser.completion.models column widths.""" - -import pytest - -from qutebrowser.completion.models.base import BaseCompletionModel -from qutebrowser.completion.models.configmodel import ( - SettingOptionCompletionModel, SettingSectionCompletionModel, - SettingValueCompletionModel) -from qutebrowser.completion.models.miscmodels import ( - CommandCompletionModel, HelpCompletionModel, QuickmarkCompletionModel, - BookmarkCompletionModel, SessionCompletionModel) -from qutebrowser.completion.models.urlmodel import UrlCompletionModel - - -CLASSES = [BaseCompletionModel, SettingOptionCompletionModel, - SettingOptionCompletionModel, SettingSectionCompletionModel, - SettingValueCompletionModel, CommandCompletionModel, - HelpCompletionModel, QuickmarkCompletionModel, - BookmarkCompletionModel, SessionCompletionModel, UrlCompletionModel] - - -@pytest.mark.parametrize("model", CLASSES) -def test_list_size(model): - """Test if there are 3 items in the COLUMN_WIDTHS property.""" - assert len(model.COLUMN_WIDTHS) == 3 - - -@pytest.mark.parametrize("model", CLASSES) -def test_column_width_sum(model): - """Test if the sum of the widths asserts to 100.""" - assert sum(model.COLUMN_WIDTHS) == 100 diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index d18e6c125..937107f75 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -34,11 +34,11 @@ class FakeCompletionModel(QStandardItemModel): """Stub for a completion model.""" - DUMB_SORT = None - - def __init__(self, kind, parent=None): + def __init__(self, kind, *pos_args, parent=None): super().__init__(parent) self.kind = kind + self.pos_args = [*pos_args] + self.dumb_sort = None class CompletionWidgetStub(QObject): @@ -74,17 +74,8 @@ def instances(monkeypatch): """Mock the instances module so get returns a fake completion model.""" # populate a model for each completion type, with a nested structure for # option and value completion - instances = {kind: FakeCompletionModel(kind) - for kind in usertypes.Completion} - instances[usertypes.Completion.option] = { - 'general': FakeCompletionModel(usertypes.Completion.option), - } - instances[usertypes.Completion.value] = { - 'general': { - 'editor': FakeCompletionModel(usertypes.Completion.value), - } - } - monkeypatch.setattr(completer, 'instances', instances) + get = lambda kind: lambda *args: FakeCompletionModel(kind, *args) + monkeypatch.setattr(completer, 'instances', get) @pytest.fixture(autouse=True) @@ -140,47 +131,52 @@ def _set_cmd_prompt(cmd, txt): cmd.setCursorPosition(txt.index('|')) -@pytest.mark.parametrize('txt, kind, pattern', [ - (':nope|', usertypes.Completion.command, 'nope'), - (':nope |', None, ''), - (':set |', usertypes.Completion.section, ''), - (':set gen|', usertypes.Completion.section, 'gen'), - (':set general |', usertypes.Completion.option, ''), - (':set what |', None, ''), - (':set general editor |', usertypes.Completion.value, ''), - (':set general editor gv|', usertypes.Completion.value, 'gv'), - (':set general editor "gvim -f"|', usertypes.Completion.value, 'gvim -f'), - (':set general editor "gvim |', usertypes.Completion.value, 'gvim'), - (':set general huh |', None, ''), - (':help |', usertypes.Completion.helptopic, ''), - (':help |', usertypes.Completion.helptopic, ''), - (':open |', usertypes.Completion.url, ''), - (':bind |', None, ''), - (':bind |', usertypes.Completion.command, ''), - (':bind foo|', usertypes.Completion.command, 'foo'), - (':bind | foo', None, ''), - (':set| general ', usertypes.Completion.command, 'set'), - (':|set general ', usertypes.Completion.command, 'set'), - (':set gene|ral ignore-case', usertypes.Completion.section, 'general'), - (':|', usertypes.Completion.command, ''), - (': |', usertypes.Completion.command, ''), - ('/|', None, ''), - (':open -t|', None, ''), - (':open --tab|', None, ''), - (':open -t |', usertypes.Completion.url, ''), - (':open --tab |', usertypes.Completion.url, ''), - (':open | -t', usertypes.Completion.url, ''), - (':tab-detach |', None, ''), - (':bind --mode=caret |', usertypes.Completion.command, ''), +@pytest.mark.parametrize('txt, kind, pattern, pos_args', [ + (':nope|', usertypes.Completion.command, 'nope', []), + (':nope |', None, '', []), + (':set |', usertypes.Completion.section, '', []), + (':set gen|', usertypes.Completion.section, 'gen', []), + (':set general |', usertypes.Completion.option, '', ['general']), + (':set what |', usertypes.Completion.option, '', ['what']), + (':set general editor |', usertypes.Completion.value, '', + ['general', 'editor']), + (':set general editor gv|', usertypes.Completion.value, 'gv', + ['general', 'editor']), + (':set general editor "gvim -f"|', usertypes.Completion.value, 'gvim -f', + ['general', 'editor']), + (':set general editor "gvim |', usertypes.Completion.value, 'gvim', + ['general', 'editor']), + (':set general huh |', usertypes.Completion.value, '', ['general', 'huh']), + (':help |', usertypes.Completion.helptopic, '', []), + (':help |', usertypes.Completion.helptopic, '', []), + (':open |', usertypes.Completion.url, '', []), + (':bind |', None, '', []), + (':bind |', usertypes.Completion.command, '', ['']), + (':bind foo|', usertypes.Completion.command, 'foo', ['']), + (':bind | foo', None, '', []), + (':set| general ', usertypes.Completion.command, 'set', []), + (':|set general ', usertypes.Completion.command, 'set', []), + (':set gene|ral ignore-case', usertypes.Completion.section, 'general', []), + (':|', usertypes.Completion.command, '', []), + (': |', usertypes.Completion.command, '', []), + ('/|', None, '', []), + (':open -t|', None, '', []), + (':open --tab|', None, '', []), + (':open -t |', usertypes.Completion.url, '', []), + (':open --tab |', usertypes.Completion.url, '', []), + (':open | -t', usertypes.Completion.url, '', []), + (':tab-detach |', None, '', []), + (':bind --mode=caret |', usertypes.Completion.command, '', + ['']), pytest.param(':bind --mode caret |', usertypes.Completion.command, - '', marks=pytest.mark.xfail(reason='issue #74')), - (':set -t -p |', usertypes.Completion.section, ''), - (':open -- |', None, ''), - (':gibberish nonesense |', None, ''), - ('/:help|', None, ''), + '', [], marks=pytest.mark.xfail(reason='issue #74')), + (':set -t -p |', usertypes.Completion.section, '', []), + (':open -- |', None, '', []), + (':gibberish nonesense |', None, '', []), + ('/:help|', None, '', []), ('::bind|', usertypes.Completion.command, ':bind'), ]) -def test_update_completion(txt, kind, pattern, status_command_stub, +def test_update_completion(txt, kind, pattern, pos_args, status_command_stub, completer_obj, completion_widget_stub): """Test setting the completion widget's model based on command text.""" # this test uses | as a placeholder for the current cursor position @@ -192,7 +188,9 @@ def test_update_completion(txt, kind, pattern, status_command_stub, if kind is None: assert args[0] is None else: - assert args[0].srcmodel.kind == kind + model = args[0].srcmodel + assert model.kind == kind + assert model.pos_args == pos_args assert args[1] == pattern diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 9a8de3cad..88f4aed20 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -71,7 +71,7 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry, def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" - model = base.BaseCompletionModel() + model = base.CompletionModel() filtermodel = sortfilter.CompletionFilterModel(model) for i in range(3): model.appendRow(QStandardItem(str(i))) @@ -82,7 +82,7 @@ def test_set_model(completionview): def test_set_pattern(completionview): - model = sortfilter.CompletionFilterModel(base.BaseCompletionModel()) + model = sortfilter.CompletionFilterModel(base.CompletionModel()) model.set_pattern = unittest.mock.Mock() completionview.set_model(model, 'foo') model.set_pattern.assert_called_with('foo') @@ -148,7 +148,7 @@ def test_completion_item_focus(which, tree, expected, completionview, qtbot): successive movement. None implies no signal should be emitted. """ - model = base.BaseCompletionModel() + model = base.CompletionModel() for catdata in tree: cat = QStandardItem() model.appendRow(cat) @@ -176,7 +176,7 @@ def test_completion_item_focus_no_model(which, completionview, qtbot): """ with qtbot.assertNotEmitted(completionview.selection_changed): completionview.completion_item_focus(which) - model = base.BaseCompletionModel() + model = base.CompletionModel() filtermodel = sortfilter.CompletionFilterModel(model, parent=completionview) completionview.set_model(filtermodel) @@ -200,7 +200,7 @@ def test_completion_show(show, rows, quick_complete, completionview, config_stub.data['completion']['show'] = show config_stub.data['completion']['quick-complete'] = quick_complete - model = base.BaseCompletionModel() + model = base.CompletionModel() for name in rows: cat = QStandardItem() model.appendRow(cat) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index ff00a11a9..9870bb4db 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -56,6 +56,9 @@ def _check_completions(model, expected): misc = actual_cat.child(j, 2) actual_item = (name.text(), desc.text(), misc.text()) assert actual_item in expected_cat + # sanity-check the column_widths + assert len(model.column_widths) == 3 + assert sum(model.column_widths) == 100 def _patch_cmdutils(monkeypatch, stubs, symbol): @@ -184,7 +187,7 @@ def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, key_config_stub.set_bindings_for('normal', {'s': 'stop', 'rr': 'roll', 'ro': 'rock'}) - model = miscmodels.CommandCompletionModel() + model = miscmodels.command() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -212,7 +215,7 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): key_config_stub.set_bindings_for('normal', {'s': 'stop', 'rr': 'roll'}) _patch_cmdutils(monkeypatch, stubs, module + '.cmdutils') _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') - model = miscmodels.HelpCompletionModel() + model = miscmodels.helptopic() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -236,7 +239,7 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" - model = miscmodels.QuickmarkCompletionModel() + model = miscmodels.quickmark() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -251,7 +254,7 @@ def test_quickmark_completion(qtmodeltester, quickmarks): def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" - model = miscmodels.BookmarkCompletionModel() + model = miscmodels.bookmark() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -275,7 +278,7 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, """ config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 2} - model = urlmodel.UrlCompletionModel() + model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -303,7 +306,7 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 2} - model = urlmodel.UrlCompletionModel() + model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -321,7 +324,7 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 2} - model = urlmodel.UrlCompletionModel() + model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -335,7 +338,7 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, def test_session_completion(qtmodeltester, session_manager_stub): session_manager_stub.sessions = ['default', '1', '2'] - model = miscmodels.SessionCompletionModel() + model = miscmodels.session() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -354,7 +357,7 @@ def test_tab_completion(qtmodeltester, fake_web_tab, app_stub, win_registry, tabbed_browser_stubs[1].tabs = [ fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0), ] - model = miscmodels.TabCompletionModel() + model = miscmodels.buffer() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -381,7 +384,7 @@ def test_tab_completion_delete(qtmodeltester, fake_web_tab, qtbot, app_stub, tabbed_browser_stubs[1].tabs = [ fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0), ] - model = miscmodels.TabCompletionModel() + model = miscmodels.buffer() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -398,7 +401,7 @@ def test_setting_section_completion(qtmodeltester, monkeypatch, stubs): _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') _patch_config_section_desc(monkeypatch, stubs, module + '.configdata.SECTION_DESC') - model = configmodel.SettingSectionCompletionModel() + model = configmodel.section() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -418,7 +421,7 @@ def test_setting_option_completion(qtmodeltester, monkeypatch, stubs, config_stub.data = {'ui': {'gesture': 'off', 'mind': 'on', 'voice': 'sometimes'}} - model = configmodel.SettingOptionCompletionModel('ui') + model = configmodel.option('ui') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -440,7 +443,7 @@ def test_setting_option_completion_valuelist(qtmodeltester, monkeypatch, stubs, 'DEFAULT': 'https://duckduckgo.com/?q={}' } } - model = configmodel.SettingOptionCompletionModel('searchengines') + model = configmodel.option('searchengines') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -454,7 +457,7 @@ def test_setting_value_completion(qtmodeltester, monkeypatch, stubs, module = 'qutebrowser.completion.models.configmodel' _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') config_stub.data = {'general': {'volume': '0'}} - model = configmodel.SettingValueCompletionModel('general', 'volume') + model = configmodel.value('general', 'volume') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -486,7 +489,7 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, key_config_stub.set_bindings_for('normal', {'s': 'stop', 'rr': 'roll', 'ro': 'rock'}) - model = miscmodels.BindCompletionModel() + model = miscmodels.bind('s') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) diff --git a/tests/unit/completion/test_sortfilter.py b/tests/unit/completion/test_sortfilter.py index 2d4a4e25d..c972a5ec9 100644 --- a/tests/unit/completion/test_sortfilter.py +++ b/tests/unit/completion/test_sortfilter.py @@ -33,7 +33,7 @@ def _create_model(data): tuple in the sub-list represents an item, and each value in the tuple represents the item data for that column """ - model = base.BaseCompletionModel() + model = base.CompletionModel() for catdata in data: cat = model.new_category('') for itemdata in catdata: @@ -74,7 +74,7 @@ def _extract_model_data(model): ('4', 'blah', False), ]) def test_filter_accepts_row(pattern, data, expected): - source_model = base.BaseCompletionModel() + source_model = base.CompletionModel() cat = source_model.new_category('test') source_model.new_item(cat, data) @@ -119,8 +119,8 @@ def test_first_last_item(tree, first, last): def test_set_source_model(): """Ensure setSourceModel sets source_model and clears the pattern.""" - model1 = base.BaseCompletionModel() - model2 = base.BaseCompletionModel() + model1 = base.CompletionModel() + model2 = base.CompletionModel() filter_model = sortfilter.CompletionFilterModel(model1) filter_model.set_pattern('foo') # sourceModel() is cached as srcmodel, so make sure both match @@ -202,7 +202,7 @@ def test_count(tree, expected): def test_set_pattern(pattern, dumb_sort, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" model = _create_model(before) - model.DUMB_SORT = dumb_sort + model.dumb_sort = dumb_sort model.columns_to_filter = filter_cols filter_model = sortfilter.CompletionFilterModel(model) filter_model.set_pattern(pattern) From b36cf0572dca4a0cd64a8a1f5539ddc61dc18372 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 19 Sep 2016 12:44:51 -0400 Subject: [PATCH 002/161] Avoid potential circular import in config.py. There was a circular import from config -> keyconf -> miscmodels -> config. This is resolved by scoping config's keyconf import to the one function that uses it. --- qutebrowser/config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index 587da214f..bb52a2417 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -38,7 +38,6 @@ from PyQt5.QtCore import pyqtSignal, QObject, QUrl, QSettings from PyQt5.QtGui import QColor from qutebrowser.config import configdata, configexc, textwrapper -from qutebrowser.config.parsers import keyconf from qutebrowser.config.parsers import ini from qutebrowser.commands import cmdexc, cmdutils from qutebrowser.utils import (message, objreg, utils, standarddir, log, @@ -181,6 +180,7 @@ def _init_key_config(parent): Args: parent: The parent to use for the KeyConfigParser. """ + from qutebrowser.config.parsers import keyconf args = objreg.get('args') try: key_config = keyconf.KeyConfigParser(standarddir.config(), 'keys.conf', From 3b30b422118c918fc19fcbc4837be28b00387fa8 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Sep 2016 06:52:52 -0400 Subject: [PATCH 003/161] Remove completion.instances, usertypes.Completion. The new completion API no longer needs either of these. Instead of referencing an enum member, cmdutils.argument.completion now points to a function that returnsthe desired completion model. This vastly simplifies the addition of new completion types. Previously it was necessary to define the new model as well as editing usertypes and completion.models.instances. Now it is only necessary to define a single function under completion.models. This is the next step of Completion Model/View Revamping (#74). --- qutebrowser/app.py | 7 +- qutebrowser/browser/commands.py | 20 ++-- qutebrowser/completion/completer.py | 8 +- qutebrowser/completion/models/instances.py | 53 ----------- qutebrowser/config/config.py | 8 +- qutebrowser/config/parsers/keyconf.py | 5 +- qutebrowser/misc/sessions.py | 11 ++- qutebrowser/utils/usertypes.py | 7 -- tests/unit/completion/test_completer.py | 102 +++++++++++---------- 9 files changed, 84 insertions(+), 137 deletions(-) delete mode 100644 qutebrowser/completion/models/instances.py diff --git a/qutebrowser/app.py b/qutebrowser/app.py index a3a855f9a..2b216ca44 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -42,7 +42,7 @@ except ImportError: import qutebrowser import qutebrowser.resources -from qutebrowser.completion.models import instances as completionmodels +from qutebrowser.completion.models import miscmodels from qutebrowser.commands import cmdutils, runners, cmdexc from qutebrowser.config import style, config, websettings, configexc from qutebrowser.browser import (urlmarks, adblock, history, browsertab, @@ -459,9 +459,6 @@ def _init_modules(args, crash_handler): diskcache = cache.DiskCache(standarddir.cache(), parent=qApp) objreg.register('cache', diskcache) - log.init.debug("Initializing completions...") - completionmodels.init() - log.init.debug("Misc initialization...") if config.get('ui', 'hide-wayland-decoration'): os.environ['QT_WAYLAND_DISABLE_WINDOWDECORATION'] = '1' @@ -753,7 +750,7 @@ class Quitter: QTimer.singleShot(0, functools.partial(qApp.exit, status)) @cmdutils.register(instance='quitter', name='wq') - @cmdutils.argument('name', completion=usertypes.Completion.sessions) + @cmdutils.argument('name', completion=miscmodels.session) def save_and_quit(self, name=sessions.default): """Save open pages and quit. diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 97a764c68..8914d4ac2 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -41,7 +41,7 @@ from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils, objreg, utils, typing) from qutebrowser.utils.usertypes import KeyMode from qutebrowser.misc import editor, guiprocess -from qutebrowser.completion.models import instances, sortfilter +from qutebrowser.completion.models import sortfilter, urlmodel, miscmodels class CommandDispatcher: @@ -284,7 +284,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', name='open', maxsplit=0, scope='window') - @cmdutils.argument('url', completion=usertypes.Completion.url) + @cmdutils.argument('url', completion=urlmodel.url) @cmdutils.argument('count', count=True) def openurl(self, url=None, implicit=False, bg=False, tab=False, window=False, count=None, secure=False, @@ -1007,7 +1007,7 @@ class CommandDispatcher: self._open(url, tab, bg, window) @cmdutils.register(instance='command-dispatcher', scope='window') - @cmdutils.argument('index', completion=usertypes.Completion.tab) + @cmdutils.argument('index', completion=miscmodels.buffer) def buffer(self, index): """Select tab by index or url/title best match. @@ -1023,7 +1023,7 @@ class CommandDispatcher: for part in index_parts: int(part) except ValueError: - model = instances.get(usertypes.Completion.tab) + model = miscmodels.buffer() sf = sortfilter.CompletionFilterModel(source=model) sf.set_pattern(index) if sf.count() > 0: @@ -1229,8 +1229,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0) - @cmdutils.argument('name', - completion=usertypes.Completion.quickmark_by_name) + @cmdutils.argument('name', completion=miscmodels.quickmark) def quickmark_load(self, name, tab=False, bg=False, window=False): """Load a quickmark. @@ -1248,8 +1247,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0) - @cmdutils.argument('name', - completion=usertypes.Completion.quickmark_by_name) + @cmdutils.argument('name', completion=miscmodels.quickmark) def quickmark_del(self, name=None): """Delete a quickmark. @@ -1311,7 +1309,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0) - @cmdutils.argument('url', completion=usertypes.Completion.bookmark_by_url) + @cmdutils.argument('url', completion=miscmodels.bookmark) def bookmark_load(self, url, tab=False, bg=False, window=False, delete=False): """Load a bookmark. @@ -1333,7 +1331,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0) - @cmdutils.argument('url', completion=usertypes.Completion.bookmark_by_url) + @cmdutils.argument('url', completion=miscmodels.bookmark) def bookmark_del(self, url=None): """Delete a bookmark. @@ -1507,7 +1505,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', name='help', scope='window') - @cmdutils.argument('topic', completion=usertypes.Completion.helptopic) + @cmdutils.argument('topic', completion=miscmodels.helptopic) def show_help(self, tab=False, bg=False, window=False, topic=None): r"""Show help about a command or setting. diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 21af74da5..bd4f43009 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -23,8 +23,8 @@ from PyQt5.QtCore import pyqtSlot, QObject, QTimer from qutebrowser.config import config from qutebrowser.commands import cmdutils, runners -from qutebrowser.utils import usertypes, log, utils -from qutebrowser.completion.models import instances, sortfilter +from qutebrowser.utils import log, utils +from qutebrowser.completion.models import sortfilter, miscmodels class Completer(QObject): @@ -72,7 +72,7 @@ class Completer(QObject): Return: A completion model or None. """ - model = instances.get(completion)(*pos_args) + model = completion(*pos_args) if model is None: return None else: @@ -96,7 +96,7 @@ class Completer(QObject): log.completion.debug("After removing flags: {}".format(before_cursor)) if not before_cursor: # '|' or 'set|' - model = instances.get(usertypes.Completion.command)() + model = miscmodels.command() return sortfilter.CompletionFilterModel(source=model, parent=self) try: cmd = cmdutils.cmd_dict[before_cursor[0]] diff --git a/qutebrowser/completion/models/instances.py b/qutebrowser/completion/models/instances.py deleted file mode 100644 index ce109ae7a..000000000 --- a/qutebrowser/completion/models/instances.py +++ /dev/null @@ -1,53 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2015-2017 Florian Bruhin (The Compiler) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Global instances of the completion models.""" - -from qutebrowser.utils import usertypes -from qutebrowser.completion.models import miscmodels, urlmodel, configmodel - - -def get(completion): - """Get a certain completion. Initializes the completion if needed.""" - if completion == usertypes.Completion.command: - return miscmodels.command - if completion == usertypes.Completion.helptopic: - return miscmodels.helptopic - if completion == usertypes.Completion.tab: - return miscmodels.buffer - if completion == usertypes.Completion.quickmark_by_name: - return miscmodels.quickmark - if completion == usertypes.Completion.bookmark_by_url: - return miscmodels.bookmark - if completion == usertypes.Completion.sessions: - return miscmodels.session - if completion == usertypes.Completion.bind: - return miscmodels.bind - if completion == usertypes.Completion.section: - return configmodel.section - if completion == usertypes.Completion.option: - return configmodel.option - if completion == usertypes.Completion.value: - return configmodel.value - if completion == usertypes.Completion.url: - return urlmodel.url - - -def init(): - pass diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index bb52a2417..dd71f5333 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -43,7 +43,7 @@ from qutebrowser.commands import cmdexc, cmdutils from qutebrowser.utils import (message, objreg, utils, standarddir, log, qtutils, error, usertypes) from qutebrowser.misc import objects -from qutebrowser.utils.usertypes import Completion +from qutebrowser.completion.models import configmodel UNSET = object() @@ -794,9 +794,9 @@ class ConfigManager(QObject): e.__class__.__name__, e)) @cmdutils.register(name='set', instance='config', star_args_optional=True) - @cmdutils.argument('section_', completion=Completion.section) - @cmdutils.argument('option', completion=Completion.option) - @cmdutils.argument('values', completion=Completion.value) + @cmdutils.argument('section_', completion=configmodel.section) + @cmdutils.argument('option', completion=configmodel.option) + @cmdutils.argument('values', completion=configmodel.value) @cmdutils.argument('win_id', win_id=True) def set_command(self, win_id, section_=None, option=None, *values, temp=False, print_=False): diff --git a/qutebrowser/config/parsers/keyconf.py b/qutebrowser/config/parsers/keyconf.py index 53f23d7c0..e4adb8676 100644 --- a/qutebrowser/config/parsers/keyconf.py +++ b/qutebrowser/config/parsers/keyconf.py @@ -27,7 +27,8 @@ from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.config import configdata, textwrapper from qutebrowser.commands import cmdutils, cmdexc -from qutebrowser.utils import log, utils, qtutils, message, usertypes +from qutebrowser.utils import log, utils, qtutils, message +from qutebrowser.completion.models import miscmodels class KeyConfigError(Exception): @@ -153,7 +154,7 @@ class KeyConfigParser(QObject): @cmdutils.register(instance='key-config', maxsplit=1, no_cmd_split=True, no_replace_variables=True) - @cmdutils.argument('command', completion=usertypes.Completion.bind) + @cmdutils.argument('command', completion=miscmodels.bind) def bind(self, key, command=None, *, mode='normal', force=False): """Bind a key to a command. diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index aa9a1be97..124b78053 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -31,10 +31,11 @@ try: except ImportError: # pragma: no cover from yaml import SafeLoader as YamlLoader, SafeDumper as YamlDumper -from qutebrowser.utils import (standarddir, objreg, qtutils, log, usertypes, - message, utils) +from qutebrowser.utils import (standarddir, objreg, qtutils, log, message, + utils) from qutebrowser.commands import cmdexc, cmdutils from qutebrowser.config import config +from qutebrowser.completion.models import miscmodels default = object() # Sentinel value @@ -433,7 +434,7 @@ class SessionManager(QObject): return sessions @cmdutils.register(instance='session-manager') - @cmdutils.argument('name', completion=usertypes.Completion.sessions) + @cmdutils.argument('name', completion=miscmodels.session) def session_load(self, name, clear=False, temp=False, force=False): """Load a session. @@ -461,7 +462,7 @@ class SessionManager(QObject): win.close() @cmdutils.register(name=['session-save', 'w'], instance='session-manager') - @cmdutils.argument('name', completion=usertypes.Completion.sessions) + @cmdutils.argument('name', completion=miscmodels.session) @cmdutils.argument('win_id', win_id=True) @cmdutils.argument('with_private', flag='p') def session_save(self, name: str = default, current=False, quiet=False, @@ -500,7 +501,7 @@ class SessionManager(QObject): message.info("Saved session {}.".format(name)) @cmdutils.register(instance='session-manager') - @cmdutils.argument('name', completion=usertypes.Completion.sessions) + @cmdutils.argument('name', completion=miscmodels.session) def session_delete(self, name, force=False): """Delete a session. diff --git a/qutebrowser/utils/usertypes.py b/qutebrowser/utils/usertypes.py index 7d31ba6ac..31f2f79cb 100644 --- a/qutebrowser/utils/usertypes.py +++ b/qutebrowser/utils/usertypes.py @@ -236,13 +236,6 @@ KeyMode = enum('KeyMode', ['normal', 'hint', 'command', 'yesno', 'prompt', 'jump_mark', 'record_macro', 'run_macro']) -# Available command completions -Completion = enum('Completion', ['command', 'section', 'option', 'value', - 'helptopic', 'quickmark_by_name', - 'bookmark_by_url', 'url', 'tab', 'sessions', - 'bind']) - - # Exit statuses for errors. Needs to be an int for sys.exit. Exit = enum('Exit', ['ok', 'reserved', 'exception', 'err_ipc', 'err_init', 'err_config', 'err_key_config'], is_int=True, start=0) diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index 937107f75..48f619104 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -26,7 +26,6 @@ from PyQt5.QtCore import QObject from PyQt5.QtGui import QStandardItemModel from qutebrowser.completion import completer -from qutebrowser.utils import usertypes from qutebrowser.commands import command, cmdutils @@ -70,30 +69,45 @@ def completer_obj(qtbot, status_command_stub, config_stub, monkeypatch, stubs, @pytest.fixture(autouse=True) -def instances(monkeypatch): - """Mock the instances module so get returns a fake completion model.""" - # populate a model for each completion type, with a nested structure for - # option and value completion - get = lambda kind: lambda *args: FakeCompletionModel(kind, *args) - monkeypatch.setattr(completer, 'instances', get) +def miscmodels_patch(mocker): + """Patch the miscmodels module to provide fake completion functions. + + Technically some of these are not part of miscmodels, but rolling them into + one module is easier and sufficient for mocking. The only one referenced + directly by Completer is miscmodels.command. + """ + m = mocker.patch('qutebrowser.completion.completer.miscmodels', + autospec=True) + m.command = lambda *args: FakeCompletionModel('command', *args) + m.helptopic = lambda *args: FakeCompletionModel('helptopic', *args) + m.quickmark = lambda *args: FakeCompletionModel('quickmark', *args) + m.bookmark = lambda *args: FakeCompletionModel('bookmark', *args) + m.session = lambda *args: FakeCompletionModel('session', *args) + m.buffer = lambda *args: FakeCompletionModel('buffer', *args) + m.bind = lambda *args: FakeCompletionModel('bind', *args) + m.url = lambda *args: FakeCompletionModel('url', *args) + m.section = lambda *args: FakeCompletionModel('section', *args) + m.option = lambda *args: FakeCompletionModel('option', *args) + m.value = lambda *args: FakeCompletionModel('value', *args) + return m @pytest.fixture(autouse=True) -def cmdutils_patch(monkeypatch, stubs): +def cmdutils_patch(monkeypatch, stubs, miscmodels_patch): """Patch the cmdutils module to provide fake commands.""" - @cmdutils.argument('section_', completion=usertypes.Completion.section) - @cmdutils.argument('option', completion=usertypes.Completion.option) - @cmdutils.argument('value', completion=usertypes.Completion.value) + @cmdutils.argument('section_', completion=miscmodels_patch.section) + @cmdutils.argument('option', completion=miscmodels_patch.option) + @cmdutils.argument('value', completion=miscmodels_patch.value) def set_command(section_=None, option=None, value=None): """docstring.""" pass - @cmdutils.argument('topic', completion=usertypes.Completion.helptopic) + @cmdutils.argument('topic', completion=miscmodels_patch.helptopic) def show_help(tab=False, bg=False, window=False, topic=None): """docstring.""" pass - @cmdutils.argument('url', completion=usertypes.Completion.url) + @cmdutils.argument('url', completion=miscmodels_patch.url) @cmdutils.argument('count', count=True) def openurl(url=None, implicit=False, bg=False, tab=False, window=False, count=None): @@ -101,7 +115,7 @@ def cmdutils_patch(monkeypatch, stubs): pass @cmdutils.argument('win_id', win_id=True) - @cmdutils.argument('command', completion=usertypes.Completion.command) + @cmdutils.argument('command', completion=miscmodels_patch.command) def bind(key, win_id, command=None, *, mode='normal', force=False): """docstring.""" pass @@ -132,49 +146,45 @@ def _set_cmd_prompt(cmd, txt): @pytest.mark.parametrize('txt, kind, pattern, pos_args', [ - (':nope|', usertypes.Completion.command, 'nope', []), + (':nope|', 'command', 'nope', []), (':nope |', None, '', []), - (':set |', usertypes.Completion.section, '', []), - (':set gen|', usertypes.Completion.section, 'gen', []), - (':set general |', usertypes.Completion.option, '', ['general']), - (':set what |', usertypes.Completion.option, '', ['what']), - (':set general editor |', usertypes.Completion.value, '', + (':set |', 'section', '', []), + (':set gen|', 'section', 'gen', []), + (':set general |', 'option', '', ['general']), + (':set what |', 'option', '', ['what']), + (':set general editor |', 'value', '', ['general', 'editor']), + (':set general editor gv|', 'value', 'gv', ['general', 'editor']), + (':set general editor "gvim -f"|', 'value', 'gvim -f', ['general', 'editor']), - (':set general editor gv|', usertypes.Completion.value, 'gv', - ['general', 'editor']), - (':set general editor "gvim -f"|', usertypes.Completion.value, 'gvim -f', - ['general', 'editor']), - (':set general editor "gvim |', usertypes.Completion.value, 'gvim', - ['general', 'editor']), - (':set general huh |', usertypes.Completion.value, '', ['general', 'huh']), - (':help |', usertypes.Completion.helptopic, '', []), - (':help |', usertypes.Completion.helptopic, '', []), - (':open |', usertypes.Completion.url, '', []), + (':set general editor "gvim |', 'value', 'gvim', ['general', 'editor']), + (':set general huh |', 'value', '', ['general', 'huh']), + (':help |', 'helptopic', '', []), + (':help |', 'helptopic', '', []), + (':open |', 'url', '', []), (':bind |', None, '', []), - (':bind |', usertypes.Completion.command, '', ['']), - (':bind foo|', usertypes.Completion.command, 'foo', ['']), + (':bind |', 'command', '', ['']), + (':bind foo|', 'command', 'foo', ['']), (':bind | foo', None, '', []), - (':set| general ', usertypes.Completion.command, 'set', []), - (':|set general ', usertypes.Completion.command, 'set', []), - (':set gene|ral ignore-case', usertypes.Completion.section, 'general', []), - (':|', usertypes.Completion.command, '', []), - (': |', usertypes.Completion.command, '', []), + (':set| general ', 'command', 'set', []), + (':|set general ', 'command', 'set', []), + (':set gene|ral ignore-case', 'section', 'general', []), + (':|', 'command', '', []), + (': |', 'command', '', []), ('/|', None, '', []), (':open -t|', None, '', []), (':open --tab|', None, '', []), - (':open -t |', usertypes.Completion.url, '', []), - (':open --tab |', usertypes.Completion.url, '', []), - (':open | -t', usertypes.Completion.url, '', []), + (':open -t |', 'url', '', []), + (':open --tab |', 'url', '', []), + (':open | -t', 'url', '', []), (':tab-detach |', None, '', []), - (':bind --mode=caret |', usertypes.Completion.command, '', - ['']), - pytest.param(':bind --mode caret |', usertypes.Completion.command, - '', [], marks=pytest.mark.xfail(reason='issue #74')), - (':set -t -p |', usertypes.Completion.section, '', []), + (':bind --mode=caret |', 'command', '', ['']), + pytest.param(':bind --mode caret |', 'command', '', [], + marks=pytest.mark.xfail(reason='issue #74')), + (':set -t -p |', 'section', '', []), (':open -- |', None, '', []), (':gibberish nonesense |', None, '', []), ('/:help|', None, '', []), - ('::bind|', usertypes.Completion.command, ':bind'), + ('::bind|', 'command', ':bind', []), ]) def test_update_completion(txt, kind, pattern, pos_args, status_command_stub, completer_obj, completion_widget_stub): From f43f78c40fd61a4cc24e088b94483ae21f43c206 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 23 Dec 2016 12:47:36 -0500 Subject: [PATCH 004/161] Implement SQL interface. When qutebrowser starts, it creates an in-memory sqlite database. One can instantiate a SqlTable to create a new table in the database. The object provides an interface to query and modify the table. This intended to serve as the base class for the quickmark, bookmark, and history manager objects in objreg. Instead of reading their data into an in-memory dict, they will read into an in-memory sql table. Eventually the completion models for history, bookmarks, and quickmarks can be replaced with SqlQuery models for faster creation and filtering. See #1765. --- qutebrowser/app.py | 5 +- qutebrowser/misc/sql.py | 166 ++++++++++++++++++++++++++++++++++++ tests/unit/misc/test_sql.py | 126 +++++++++++++++++++++++++++ 3 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 qutebrowser/misc/sql.py create mode 100644 tests/unit/misc/test_sql.py diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 2b216ca44..8b2aa7d69 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -53,7 +53,7 @@ from qutebrowser.browser.webkit.network import networkmanager from qutebrowser.keyinput import macros from qutebrowser.mainwindow import mainwindow, prompt from qutebrowser.misc import (readline, ipc, savemanager, sessions, - crashsignal, earlyinit, objects) + crashsignal, earlyinit, objects, sql) from qutebrowser.misc import utilcmds # pylint: disable=unused-import from qutebrowser.utils import (log, version, message, utils, qtutils, urlutils, objreg, usertypes, standarddir, error, debug) @@ -423,6 +423,9 @@ def _init_modules(args, crash_handler): config.init(qApp) save_manager.init_autosave() + log.init.debug("Initializing sql...") + sql.init() + log.init.debug("Initializing web history...") history.init(qApp) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py new file mode 100644 index 000000000..fd7257bdb --- /dev/null +++ b/qutebrowser/misc/sql.py @@ -0,0 +1,166 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Provides access to an in-memory sqlite database.""" + +from PyQt5.QtCore import QObject, pyqtSignal +from PyQt5.QtSql import QSqlDatabase, QSqlQuery + +from qutebrowser.utils import log + + +def init(): + """Initialize the SQL database connection.""" + database = QSqlDatabase.addDatabase('QSQLITE') + # In-memory database, see https://sqlite.org/inmemorydb.html + database.setDatabaseName(':memory:') + if not database.open(): + raise SqlException("Failed to open in-memory sqlite database") + + +def close(): + """Close the SQL connection.""" + QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) + + +def _run_query(querystr, *values): + """Run the given SQL query string on the database. + + Args: + values: positional parameter bindings. + """ + log.completion.debug('Running SQL query: "{}"'.format(querystr)) + database = QSqlDatabase.database() + query = QSqlQuery(database) + query.prepare(querystr) + for val in values: + query.addBindValue(val) + log.completion.debug('Query bindings: {}'.format(query.boundValues())) + if not query.exec_(): + raise SqlException('Failed to exec query "{}": "{}"'.format( + querystr, query.lastError().text())) + return query + + +class SqlTable(QObject): + + """Interface to a sql table. + + Attributes: + _name: Name of the SQL table this wraps. + _primary_key: The primary key of the table. + + Signals: + changed: Emitted when the table is modified. + """ + + changed = pyqtSignal() + + def __init__(self, name, fields, primary_key, parent=None): + """Create a new table in the sql database. + + Raises SqlException if the table already exists. + + Args: + name: Name of the table. + fields: A list of field names. + primary_key: Name of the field to serve as the primary key. + """ + super().__init__(parent) + self._name = name + self._primary_key = primary_key + _run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))" + .format(name, ','.join(fields), primary_key)) + + def __iter__(self): + """Iterate rows in the table.""" + result = _run_query("SELECT * FROM {}".format(self._name)) + while result.next(): + rec = result.record() + yield tuple(rec.value(i) for i in range(rec.count())) + + def __contains__(self, key): + """Return whether the table contains the matching item. + + Args: + key: Primary key value to search for. + """ + query = _run_query("SELECT * FROM {} where {} = ?" + .format(self._name, self._primary_key), key) + return query.next() + + def __len__(self): + """Return the count of rows in the table.""" + result = _run_query("SELECT count(*) FROM {}".format(self._name)) + result.next() + return result.value(0) + + def __getitem__(self, key): + """Retrieve the row matching the given key. + + Args: + key: Primary key value to fetch. + """ + result = _run_query("SELECT * FROM {} where {} = ?" + .format(self._name, self._primary_key), key) + result.next() + rec = result.record() + return tuple(rec.value(i) for i in range(rec.count())) + + def delete(self, value, field=None): + """Remove all rows for which `field` equals `value`. + + Args: + value: Key value to delete. + field: Field to use as the key, defaults to the primary key. + + Return: + The number of rows deleted. + """ + field = field or self._primary_key + query = _run_query("DELETE FROM {} where {} = ?" + .format(self._name, field), value) + if not query.numRowsAffected(): + raise KeyError('No row with {} = "{}"'.format(field, value)) + self.changed.emit() + + def insert(self, *values, replace=False): + """Append a row to the table. + + Args: + values: Values in the order fields were given on table creation. + replace: If true, allow inserting over an existing primary key. + """ + cmd = "REPLACE" if replace else "INSERT" + paramstr = ','.join(['?'] * len(values)) + _run_query("{} INTO {} values({})" + .format(cmd, self._name, paramstr), *values) + self.changed.emit() + + def delete_all(self): + """Remove all row from the table.""" + _run_query("DELETE FROM {}".format(self._name)) + self.changed.emit() + + +class SqlException(Exception): + + """Raised on an error interacting with the SQL database.""" + + pass diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py new file mode 100644 index 000000000..175f2f97e --- /dev/null +++ b/tests/unit/misc/test_sql.py @@ -0,0 +1,126 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Test the SQL API.""" + +import pytest +from qutebrowser.misc import sql + + +@pytest.fixture(autouse=True) +def init(): + sql.init() + yield + sql.close() + + +def test_init(): + sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + with pytest.raises(sql.SqlException): + # table name collision on 'Foo' + sql.SqlTable('Foo', ['foo', 'bar'], primary_key='foo') + + +def test_insert(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + with qtbot.waitSignal(table.changed): + table.insert('one', 1, False) + with qtbot.waitSignal(table.changed): + table.insert('wan', 1, False) + with pytest.raises(sql.SqlException): + # duplicate primary key + table.insert('one', 1, False) + + +def test_iter(): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + table.insert('nine', 9, False) + table.insert('thirteen', 13, True) + assert list(table) == [('one', 1, False), + ('nine', 9, False), + ('thirteen', 13, True)] + + +def test_replace(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + with qtbot.waitSignal(table.changed): + table.insert('one', 1, True, replace=True) + assert list(table) == [('one', 1, True)] + + +def test_delete(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + table.insert('nine', 9, False) + table.insert('thirteen', 13, True) + with pytest.raises(KeyError): + table.delete('nope') + with qtbot.waitSignal(table.changed): + table.delete('thirteen') + assert list(table) == [('one', 1, False), ('nine', 9, False)] + with qtbot.waitSignal(table.changed): + table.delete(False, field='lucky') + assert not list(table) == [('thirteen', 13, True)] + + +def test_len(): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + assert len(table) == 0 + table.insert('one', 1, False) + assert len(table) == 1 + table.insert('nine', 9, False) + assert len(table) == 2 + table.insert('thirteen', 13, True) + assert len(table) == 3 + + +def test_contains(): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + table.insert('nine', 9, False) + table.insert('thirteen', 13, True) + assert 'oone' not in table + assert 'ninee' not in table + assert 1 not in table + assert '*' not in table + assert 'one' in table + assert 'nine' in table + assert 'thirteen' in table + + +def test_index(): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + table.insert('nine', 9, False) + table.insert('thirteen', 13, True) + assert table['one'] == ('one', 1, False) + assert table['nine'] == ('nine', 9, False) + assert table['thirteen'] == ('thirteen', 13, True) + + +def test_delete_all(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + table.insert('nine', 9, False) + table.insert('thirteen', 13, True) + with qtbot.waitSignal(table.changed): + table.delete_all() + assert list(table) == [] From 9477a2eeb2e779d136d9d2d212c2db8a15d905ea Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 15 Jan 2017 21:42:35 -0500 Subject: [PATCH 005/161] Use SQL for history storage. The browser-wide in-memory web history is now stored in an in-memory sql database instead of a python dict. Long-term storage is not affected, it is still persisted in a text file of the same format. This will set the stage for SQL-based history completion. See #1765. --- qutebrowser/browser/history.py | 61 +++-------- qutebrowser/browser/webkit/webkithistory.py | 2 +- tests/unit/browser/webkit/test_history.py | 109 +++++++++----------- 3 files changed, 65 insertions(+), 107 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index aaad08fb3..a1fc7dfa2 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -27,7 +27,7 @@ from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject from qutebrowser.commands import cmdutils from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils, usertypes, message) -from qutebrowser.misc import lineparser, objects +from qutebrowser.misc import lineparser, objects, sql class Entry: @@ -103,19 +103,16 @@ class Entry: return cls(atime, url, title, redirect=redirect) -class WebHistory(QObject): +class WebHistory(sql.SqlTable): """The global history of visited pages. This is a little more complex as you'd expect so the history can be read from disk async while new history is already arriving. - self.history_dict is the main place where the history is stored, in an - OrderedDict (sorted by time) of URL strings mapped to Entry objects. - While reading from disk is still ongoing, the history is saved in - self._temp_history instead, and then appended to self.history_dict once - that's fully populated. + self._temp_history instead, and then inserted into the sql table once + the async read completes. All history which is new in this session (rather than read from disk from a previous browsing session) is also stored in self._new_history. @@ -123,52 +120,34 @@ class WebHistory(QObject): disk, so we can always append to the existing data. Attributes: - history_dict: An OrderedDict of URLs read from the on-disk history. _lineparser: The AppendLineParser used to save the history. _new_history: A list of Entry items of the current session. _saved_count: How many HistoryEntries have been written to disk. _initial_read_started: Whether async_read was called. _initial_read_done: Whether async_read has completed. - _temp_history: OrderedDict of temporary history entries before - async_read was called. + _temp_history: List of history entries from before async_read finished. Signals: - add_completion_item: Emitted before a new Entry is added. - Used to sync with the completion. - arg: The new Entry. - item_added: Emitted after a new Entry is added. - Used to tell the savemanager that the history is dirty. - arg: The new Entry. cleared: Emitted after the history is cleared. """ - add_completion_item = pyqtSignal(Entry) - item_added = pyqtSignal(Entry) cleared = pyqtSignal() async_read_done = pyqtSignal() def __init__(self, hist_dir, hist_name, parent=None): - super().__init__(parent) + super().__init__("History", ['url', 'title', 'atime', 'redirect'], + primary_key='url', parent=parent) self._initial_read_started = False self._initial_read_done = False self._lineparser = lineparser.AppendLineParser(hist_dir, hist_name, parent=self) - self.history_dict = collections.OrderedDict() - self._temp_history = collections.OrderedDict() + self._temp_history = [] self._new_history = [] self._saved_count = 0 - objreg.get('save-manager').add_saveable( - 'history', self.save, self.item_added) def __repr__(self): return utils.get_repr(self, length=len(self)) - def __iter__(self): - return iter(self.history_dict.values()) - - def __len__(self): - return len(self.history_dict) - def async_read(self): """Read the initial history.""" if self._initial_read_started: @@ -200,21 +179,18 @@ class WebHistory(QObject): self._initial_read_done = True self.async_read_done.emit() + objreg.get('save-manager').add_saveable( + 'history', self.save, self.changed) - for entry in self._temp_history.values(): + for entry in self._temp_history: self._add_entry(entry) self._new_history.append(entry) - if not entry.redirect: - self.add_completion_item.emit(entry) self._temp_history.clear() - def _add_entry(self, entry, target=None): - """Add an entry to self.history_dict or another given OrderedDict.""" - if target is None: - target = self.history_dict - url_str = entry.url_str() - target[url_str] = entry - target.move_to_end(url_str) + def _add_entry(self, entry): + """Add an entry to the in-memory database.""" + self.insert(entry.url_str(), entry.title, entry.atime, entry.redirect, + replace=True) def get_recent(self): """Get the most recent history entries.""" @@ -247,7 +223,7 @@ class WebHistory(QObject): def _do_clear(self): self._lineparser.clear() - self.history_dict.clear() + self.delete_all() self._temp_history.clear() self._new_history.clear() self._saved_count = 0 @@ -291,11 +267,8 @@ class WebHistory(QObject): if self._initial_read_done: self._add_entry(entry) self._new_history.append(entry) - self.item_added.emit(entry) - if not entry.redirect: - self.add_completion_item.emit(entry) else: - self._add_entry(entry, target=self._temp_history) + self._temp_history.append(entry) def init(parent=None): diff --git a/qutebrowser/browser/webkit/webkithistory.py b/qutebrowser/browser/webkit/webkithistory.py index 0f9d64460..453a11883 100644 --- a/qutebrowser/browser/webkit/webkithistory.py +++ b/qutebrowser/browser/webkit/webkithistory.py @@ -48,7 +48,7 @@ class WebHistoryInterface(QWebHistoryInterface): Return: True if the url is in the history, False otherwise. """ - return url_string in self._history.history_dict + return url_string in self._history def init(history): diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index f40e41c2c..6ad461692 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -27,24 +27,33 @@ from hypothesis import strategies from PyQt5.QtCore import QUrl from qutebrowser.browser import history -from qutebrowser.utils import objreg, urlutils, usertypes +from qutebrowser.utils import objreg, urlutils +from qutebrowser.misc import sql -class FakeWebHistory: - - """A fake WebHistory object.""" - - def __init__(self, history_dict): - self.history_dict = history_dict +@pytest.fixture(autouse=True) +def prerequisites(config_stub, fake_save_manager): + """Make sure everything is ready to initialize a WebHistory.""" + config_stub.data = {'general': {'private-browsing': False}} + sql.init() + yield + sql.close() @pytest.fixture() -def hist(tmpdir, fake_save_manager): +def hist(tmpdir): return history.WebHistory(hist_dir=str(tmpdir), hist_name='history') -def test_async_read_twice(monkeypatch, qtbot, tmpdir, caplog, +def test_register_saveable(monkeypatch, qtbot, tmpdir, caplog, fake_save_manager): + (tmpdir / 'filled-history').write('12345 http://example.com/ title') + hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') + list(hist.async_read()) + assert fake_save_manager.add_saveable.called + + +def test_async_read_twice(monkeypatch, qtbot, tmpdir, caplog): (tmpdir / 'filled-history').write('\n'.join([ '12345 http://example.com/ title', '67890 http://example.com/', @@ -55,42 +64,30 @@ def test_async_read_twice(monkeypatch, qtbot, tmpdir, caplog, with pytest.raises(StopIteration): next(hist.async_read()) expected = "Ignoring async_read() because reading is started." - assert len(caplog.records) == 1 - assert caplog.records[0].msg == expected + assert expected in (record.msg for record in caplog.records) @pytest.mark.parametrize('redirect', [True, False]) def test_adding_item_during_async_read(qtbot, hist, redirect): """Check what happens when adding URL while reading the history.""" - url = QUrl('http://www.example.com/') + url = 'http://www.example.com/' + hist.add_url(QUrl(url), redirect=redirect, atime=12345) - with qtbot.assertNotEmitted(hist.add_completion_item), \ - qtbot.assertNotEmitted(hist.item_added): - hist.add_url(url, redirect=redirect, atime=12345) - - if redirect: - with qtbot.assertNotEmitted(hist.add_completion_item): - with qtbot.waitSignal(hist.async_read_done): - list(hist.async_read()) - else: - with qtbot.waitSignals([hist.add_completion_item, - hist.async_read_done], order='strict'): - list(hist.async_read()) + with qtbot.waitSignal(hist.async_read_done): + list(hist.async_read()) assert not hist._temp_history - - expected = history.Entry(url=url, atime=12345, redirect=redirect, title="") - assert list(hist.history_dict.values()) == [expected] + assert list(hist) == [(url, '', 12345, redirect)] def test_iter(hist): list(hist.async_read()) - url = QUrl('http://www.example.com/') + urlstr = 'http://www.example.com/' + url = QUrl(urlstr) hist.add_url(url, atime=12345) - entry = history.Entry(url=url, atime=12345, redirect=False, title="") - assert list(hist) == [entry] + assert list(hist) == [(urlstr, '', 12345, False)] def test_len(hist): @@ -110,38 +107,39 @@ def test_len(hist): ' ', '', ]) -def test_read(hist, tmpdir, line): +def test_read(tmpdir, line): (tmpdir / 'filled-history').write(line + '\n') hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') list(hist.async_read()) -def test_updated_entries(hist, tmpdir): +def test_updated_entries(tmpdir): (tmpdir / 'filled-history').write('12345 http://example.com/\n' '67890 http://example.com/\n') hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') list(hist.async_read()) - assert hist.history_dict['http://example.com/'].atime == 67890 + assert hist['http://example.com/'] == ('http://example.com/', '', 67890, + False) hist.add_url(QUrl('http://example.com/'), atime=99999) - assert hist.history_dict['http://example.com/'].atime == 99999 + assert hist['http://example.com/'] == ('http://example.com/', '', 99999, + False) -def test_invalid_read(hist, tmpdir, caplog): +def test_invalid_read(tmpdir, caplog): (tmpdir / 'filled-history').write('foobar\n12345 http://example.com/') hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') with caplog.at_level(logging.WARNING): list(hist.async_read()) - entries = list(hist.history_dict.values()) + entries = list(hist) assert len(entries) == 1 - assert len(caplog.records) == 1 msg = "Invalid history entry 'foobar': 2 or 3 fields expected!" - assert caplog.records[0].msg == msg + assert msg in (rec.msg for rec in caplog.records) -def test_get_recent(hist, tmpdir): +def test_get_recent(tmpdir): (tmpdir / 'filled-history').write('12345 http://example.com/') hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') list(hist.async_read()) @@ -154,7 +152,7 @@ def test_get_recent(hist, tmpdir): assert lines == expected -def test_save(hist, tmpdir): +def test_save(tmpdir): hist_file = tmpdir / 'filled-history' hist_file.write('12345 http://example.com/\n') @@ -177,7 +175,7 @@ def test_save(hist, tmpdir): assert lines == expected -def test_clear(qtbot, hist, tmpdir): +def test_clear(qtbot, tmpdir): hist_file = tmpdir / 'filled-history' hist_file.write('12345 http://example.com/\n') @@ -190,7 +188,6 @@ def test_clear(qtbot, hist, tmpdir): hist._do_clear() assert not hist_file.read() - assert not hist.history_dict assert not hist._new_history hist.add_url(QUrl('http://www.the-compiler.org/'), atime=67890) @@ -204,24 +201,17 @@ def test_add_item(qtbot, hist): list(hist.async_read()) url = 'http://www.example.com/' - with qtbot.waitSignals([hist.add_completion_item, hist.item_added], - order='strict'): - hist.add_url(QUrl(url), atime=12345, title="the title") + hist.add_url(QUrl(url), atime=12345, title="the title") - entry = history.Entry(url=QUrl(url), redirect=False, atime=12345, - title="the title") - assert hist.history_dict[url] == entry + assert hist[url] == (url, 'the title', 12345, False) def test_add_item_redirect(qtbot, hist): list(hist.async_read()) url = 'http://www.example.com/' - with qtbot.assertNotEmitted(hist.add_completion_item): - with qtbot.waitSignal(hist.item_added): - hist.add_url(QUrl(url), redirect=True, atime=12345) + hist.add_url(QUrl(url), redirect=True, atime=12345) - entry = history.Entry(url=QUrl(url), redirect=True, atime=12345, title="") - assert hist.history_dict[url] == entry + assert hist[url] == (url, '', 12345, True) def test_add_item_redirect_update(qtbot, tmpdir, fake_save_manager): @@ -233,12 +223,9 @@ def test_add_item_redirect_update(qtbot, tmpdir, fake_save_manager): hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') list(hist.async_read()) - with qtbot.assertNotEmitted(hist.add_completion_item): - with qtbot.waitSignal(hist.item_added): - hist.add_url(QUrl(url), redirect=True, atime=67890) + hist.add_url(QUrl(url), redirect=True, atime=67890) - entry = history.Entry(url=QUrl(url), redirect=True, atime=67890, title="") - assert hist.history_dict[url] == entry + assert hist[url] == (url, '', 67890, True) @pytest.mark.parametrize('line, expected', [ @@ -333,8 +320,7 @@ def hist_interface(): entry = history.Entry(atime=0, url=QUrl('http://www.example.com/'), title='example') history_dict = {'http://www.example.com/': entry} - fake_hist = FakeWebHistory(history_dict) - interface = webkithistory.WebHistoryInterface(fake_hist) + interface = webkithistory.WebHistoryInterface(history_dict) QWebHistoryInterface.setDefaultInterface(interface) yield QWebHistoryInterface.setDefaultInterface(None) @@ -349,7 +335,7 @@ def test_history_interface(qtbot, webview, hist_interface): @pytest.mark.parametrize('backend', [usertypes.Backend.QtWebEngine, usertypes.Backend.QtWebKit]) -def test_init(backend, qapp, tmpdir, monkeypatch, fake_save_manager): +def test_init(backend, qapp, tmpdir, monkeypatch): if backend == usertypes.Backend.QtWebKit: pytest.importorskip('PyQt5.QtWebKitWidgets') else: @@ -379,5 +365,4 @@ def test_init(backend, qapp, tmpdir, monkeypatch, fake_save_manager): # before (so we need to test webengine before webkit) assert default_interface is None - assert fake_save_manager.add_saveable.called objreg.delete('web-history') From 93d81d96ce3314866a260d9a1d71b008674944e4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 22 Jan 2017 07:14:42 -0500 Subject: [PATCH 006/161] Use SQL for quickmark/bookmark storage. Store quickmarks and bookmarks in an in-memory sql database instead of a python dict. Long-term storage is not affected, bookmarks and quickmarks are still persisted in a text file. The added and deleted signals were removed, as once sql completion models are used the models will no longer need to update themselves. This will set the stage for SQL-based history completion. See #1765. --- qutebrowser/browser/commands.py | 14 ++-- qutebrowser/browser/urlmarks.py | 112 ++++++++++---------------------- 2 files changed, 42 insertions(+), 84 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 8914d4ac2..9d19914f8 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1260,13 +1260,15 @@ class CommandDispatcher: if name is None: url = self._current_url() try: - name = quickmark_manager.get_by_qurl(url) - except urlmarks.DoesNotExistError as e: + quickmark_manager.delete_by_qurl(url) + except KeyError as e: raise cmdexc.CommandError(str(e)) - try: - quickmark_manager.delete(name) - except KeyError: - raise cmdexc.CommandError("Quickmark '{}' not found!".format(name)) + else: + try: + quickmark_manager.delete(name) + except KeyError: + raise cmdexc.CommandError( + "Quickmark '{}' not found!".format(name)) @cmdutils.register(instance='command-dispatcher', scope='window') def bookmark_add(self, url=None, title=None, toggle=False): diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index 013de408c..db5e14153 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -31,12 +31,12 @@ import os.path import functools import collections -from PyQt5.QtCore import pyqtSignal, QUrl, QObject +from PyQt5.QtCore import QUrl, QObject from qutebrowser.utils import (message, usertypes, qtutils, urlutils, standarddir, objreg, log) from qutebrowser.commands import cmdutils -from qutebrowser.misc import lineparser +from qutebrowser.misc import lineparser, sql class Error(Exception): @@ -53,13 +53,6 @@ class InvalidUrlError(Error): pass -class DoesNotExistError(Error): - - """Exception emitted when a given URL does not exist.""" - - pass - - class AlreadyExistsError(Error): """Exception emitted when a given URL does already exist.""" @@ -67,29 +60,18 @@ class AlreadyExistsError(Error): pass -class UrlMarkManager(QObject): +class UrlMarkManager(sql.SqlTable): """Base class for BookmarkManager and QuickmarkManager. Attributes: marks: An OrderedDict of all quickmarks/bookmarks. _lineparser: The LineParser used for the marks - - Signals: - changed: Emitted when anything changed. - added: Emitted when a new quickmark/bookmark was added. - removed: Emitted when an existing quickmark/bookmark was removed. """ - changed = pyqtSignal() - added = pyqtSignal(str, str) - removed = pyqtSignal(str) - - def __init__(self, parent=None): - """Initialize and read quickmarks.""" - super().__init__(parent) - - self.marks = collections.OrderedDict() + def __init__(self, name, fields, primary_key, parent=None): + """Initialize and read marks.""" + super().__init__(name, fields, primary_key, parent) self._init_lineparser() for line in self._lineparser: @@ -110,32 +92,21 @@ class UrlMarkManager(QObject): def save(self): """Save the marks to disk.""" - self._lineparser.data = [' '.join(tpl) for tpl in self.marks.items()] + self._lineparser.data = [' '.join(tpl) for tpl in self] self._lineparser.save() - def delete(self, key): - """Delete a quickmark/bookmark. - - Args: - key: The key to delete (name for quickmarks, URL for bookmarks.) - """ - del self.marks[key] - self.changed.emit() - self.removed.emit(key) - class QuickmarkManager(UrlMarkManager): """Manager for quickmarks. - The primary key for quickmarks is their *name*, this means: - - - self.marks maps names to URLs. - - changed gets emitted with the name as first argument and the URL as - second argument. - - removed gets emitted with the name as argument. + The primary key for quickmarks is their *name*. """ + def __init__(self, parent=None): + super().__init__('Quickmarks', ['name', 'url'], primary_key='name', + parent=parent) + def _init_lineparser(self): self._lineparser = lineparser.LineParser( standarddir.config(), 'quickmarks', parent=self) @@ -151,7 +122,7 @@ class QuickmarkManager(UrlMarkManager): except ValueError: message.error("Invalid quickmark '{}'".format(line)) else: - self.marks[key] = url + self.insert(key, url) def prompt_save(self, url): """Prompt for a new quickmark name to be added and add it. @@ -191,41 +162,30 @@ class QuickmarkManager(UrlMarkManager): def set_mark(): """Really set the quickmark.""" - self.marks[name] = url - self.changed.emit() - self.added.emit(name, url) + self.insert(name, url) log.misc.debug("Added quickmark {} for {}".format(name, url)) - if name in self.marks: + if name in self: message.confirm_async( title="Override existing quickmark?", yes_action=set_mark, default=True) else: set_mark() - def get_by_qurl(self, url): - """Look up a quickmark by QUrl, returning its name. - - Takes O(n) time, where n is the number of quickmarks. - Use a name instead where possible. - """ + def delete_by_qurl(self, url): + """Look up a quickmark by QUrl, returning its name.""" qtutils.ensure_valid(url) urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) - try: - index = list(self.marks.values()).index(urlstr) - key = list(self.marks.keys())[index] - except ValueError: - raise DoesNotExistError( - "Quickmark for '{}' not found!".format(urlstr)) - return key + self.delete(urlstr, field='url') + except KeyError: + raise KeyError("Quickmark for '{}' not found!".format(urlstr)) def get(self, name): """Get the URL of the quickmark named name as a QUrl.""" - if name not in self.marks: - raise DoesNotExistError( - "Quickmark '{}' does not exist!".format(name)) - urlstr = self.marks[name] + if name not in self: + raise KeyError("Quickmark '{}' does not exist!".format(name)) + urlstr = self[name] try: url = urlutils.fuzzy_url(urlstr, do_search=False) except urlutils.InvalidUrlError as e: @@ -238,14 +198,13 @@ class BookmarkManager(UrlMarkManager): """Manager for bookmarks. - The primary key for bookmarks is their *url*, this means: - - - self.marks maps URLs to titles. - - changed gets emitted with the URL as first argument and the title as - second argument. - - removed gets emitted with the URL as argument. + The primary key for bookmarks is their *url*. """ + def __init__(self, parent=None): + super().__init__('Bookmarks', ['url', 'title'], primary_key='url', + parent=parent) + def _init_lineparser(self): bookmarks_directory = os.path.join(standarddir.config(), 'bookmarks') if not os.path.isdir(bookmarks_directory): @@ -262,10 +221,9 @@ class BookmarkManager(UrlMarkManager): def _parse_line(self, line): parts = line.split(maxsplit=1) - if len(parts) == 2: - self.marks[parts[0]] = parts[1] - elif len(parts) == 1: - self.marks[parts[0]] = '' + urlstr = parts[0] + title = parts[1] if len(parts) == 2 else '' + self.insert(urlstr, title) def add(self, url, title, *, toggle=False): """Add a new bookmark. @@ -286,14 +244,12 @@ class BookmarkManager(UrlMarkManager): urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) - if urlstr in self.marks: + if urlstr in self: if toggle: - del self.marks[urlstr] + self.delete(urlstr) return False else: raise AlreadyExistsError("Bookmark already exists!") else: - self.marks[urlstr] = title - self.changed.emit() - self.added.emit(title, urlstr) + self.insert(urlstr, title) return True From 6e1ea89ca17b0dc0a77ffcfd74f359ef1337088d Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 22 Jan 2017 07:18:41 -0500 Subject: [PATCH 007/161] Implement SQL completion model. This model wraps one or more SQL tables and exposes data in a tiered manner that can be consumed by the tree view used for completion. --- qutebrowser/completion/completer.py | 4 +- qutebrowser/completion/models/sqlmodel.py | 218 ++++++++++++++++++++++ tests/unit/completion/test_sqlmodel.py | 204 ++++++++++++++++++++ 3 files changed, 424 insertions(+), 2 deletions(-) create mode 100644 qutebrowser/completion/models/sqlmodel.py create mode 100644 tests/unit/completion/test_sqlmodel.py diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index bd4f43009..8742156a3 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -73,8 +73,8 @@ class Completer(QObject): A completion model or None. """ model = completion(*pos_args) - if model is None: - return None + if model is None or hasattr(model, 'set_pattern'): + return model else: return sortfilter.CompletionFilterModel(source=model, parent=self) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py new file mode 100644 index 000000000..a5a9bc99a --- /dev/null +++ b/qutebrowser/completion/models/sqlmodel.py @@ -0,0 +1,218 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""A completion model backed by SQL tables.""" + +import re + +from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel +from PyQt5.QtSql import QSqlTableModel, QSqlDatabase, QSqlQuery + +from qutebrowser.utils import usertypes, log + + +class SqlCompletionModel(QAbstractItemModel): + + """A sqlite-based model that provides data for the CompletionView. + + This model is a wrapper around one or more sql tables. The tables are all + stored in a single database in qutebrowser's cache directory. + + Top level indices represent categories, each of which is backed by a single + table. Child indices represent rows of those tables. + + Class Attributes: + COLUMN_WIDTHS: The width percentages of the columns used in the + completion view. + + Attributes: + column_widths: The width percentages of the columns used in the + completion view. + columns_to_filter: A list of indices of columns to apply the filter to. + pattern: Current filter pattern, used for highlighting. + _categories: The category tables. + """ + + def __init__(self, column_widths=(30, 70, 0), columns_to_filter=None, + parent=None): + super().__init__(parent) + self.columns_to_filter = columns_to_filter or [0] + self.column_widths = column_widths + self._categories = [] + self.srcmodel = self # TODO: dummy for compat with old API + self.pattern = '' + + def new_category(self, name, sort_by=None, sort_order=Qt.AscendingOrder): + """Create a new completion category and add it to this model. + + Args: + name: Name of category, and the table in the database. + sort_by: The name of the field to sort by, or None for no sorting. + sort_order: Sorting order, if sort_by is non-None. + + Return: A new CompletionCategory. + """ + database = QSqlDatabase.database() + cat = QSqlTableModel(parent=self, db=database) + cat.setTable(name) + if sort_by: + cat.setSort(cat.fieldIndex(sort_by), sort_order) + cat.select() + self._categories.append(cat) + return cat + + def delete_cur_item(self, completion): + """Delete the selected item.""" + raise NotImplementedError + + def data(self, index, role=Qt.DisplayRole): + """Return the item data for index. + + Override QAbstractItemModel::data. + + Args: + index: The QModelIndex to get item flags for. + + Return: The item data, or None on an invalid index. + """ + if not index.isValid() or role != Qt.DisplayRole: + return + if not index.parent().isValid(): + if index.column() == 0: + return self._categories[index.row()].tableName() + else: + table = self._categories[index.parent().row()] + idx = table.index(index.row(), index.column()) + return table.data(idx) + + def flags(self, index): + """Return the item flags for index. + + Override QAbstractItemModel::flags. + + Return: The item flags, or Qt.NoItemFlags on error. + """ + if not index.isValid(): + return + if index.parent().isValid(): + # item + return (Qt.ItemIsEnabled | Qt.ItemIsSelectable | + Qt.ItemNeverHasChildren) + else: + # category + return Qt.NoItemFlags + + def index(self, row, col, parent=QModelIndex()): + """Get an index into the model. + + Override QAbstractItemModel::index. + + Return: A QModelIndex. + """ + if (row < 0 or row >= self.rowCount(parent) or + col < 0 or col >= self.columnCount(parent)): + return QModelIndex() + if parent.isValid(): + if parent.column() != 0: + return QModelIndex() + # store a pointer to the parent table in internalPointer + return self.createIndex(row, col, self._categories[parent.row()]) + return self.createIndex(row, col, None) + + def parent(self, index): + """Get an index to the parent of the given index. + + Override QAbstractItemModel::parent. + + Args: + index: The QModelIndex to get the parent index for. + """ + parent_table = index.internalPointer() + if not parent_table: + # categories have no parent + return QModelIndex() + row = self._categories.index(parent_table) + return self.createIndex(row, 0, None) + + def rowCount(self, parent=QModelIndex()): + if not parent.isValid(): + # top-level + return len(self._categories) + elif parent.internalPointer() or parent.column() != 0: + # item or nonzero category column (only first col has children) + return 0 + else: + # category + return self._categories[parent.row()].rowCount() + + def columnCount(self, parent=QModelIndex()): + # pylint: disable=unused-argument + return 3 + + def count(self): + """Return the count of non-category items.""" + return sum(t.rowCount() for t in self._categories) + + def set_pattern(self, pattern): + """Set the filter pattern for all category tables. + + This will apply to the fields indicated in columns_to_filter. + + Args: + pattern: The filter pattern to set. + """ + # TODO: should pattern be saved in the view layer instead? + self.pattern = pattern + # escape to treat a user input % or _ as a literal, not a wildcard + pattern = pattern.replace('%', '\\%') + pattern = pattern.replace('_', '\\_') + # treat spaces as wildcards to match any of the typed words + pattern = re.sub(r' +', '%', pattern) + for t in self._categories: + fields = (t.record().fieldName(i) for i in self.columns_to_filter) + query = ' or '.join("{} like '%{}%' escape '\\'" + .format(field, pattern) + for field in fields) + log.completion.debug("Setting filter = '{}' for table '{}'" + .format(query, t.tableName())) + t.setFilter(query) + + def first_item(self): + """Return the index of the first child (non-category) in the model.""" + for row, table in enumerate(self._categories): + if table.rowCount() > 0: + parent = self.index(row, 0) + return self.index(0, 0, parent) + return QModelIndex() + + def last_item(self): + """Return the index of the last child (non-category) in the model.""" + for row, table in reversed(list(enumerate(self._categories))): + childcount = table.rowCount() + if childcount > 0: + parent = self.index(row, 0) + return self.index(childcount - 1, 0, parent) + return QModelIndex() + + +class SqlException(Exception): + + """Raised on an error interacting with the SQL database.""" + + pass diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py new file mode 100644 index 000000000..6b8f01f4d --- /dev/null +++ b/tests/unit/completion/test_sqlmodel.py @@ -0,0 +1,204 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Tests for the base sql completion model.""" + +import pytest +from PyQt5.QtCore import Qt + +from qutebrowser.misc import sql +from qutebrowser.completion.models import sqlmodel + + +@pytest.fixture(autouse=True) +def init(): + sql.init() + yield + sql.close() + + +def _check_model(model, expected): + """Check that a model contains the expected items in the given order. + + Args: + expected: A list of form + [ + (cat, [(name, desc, misc), (name, desc, misc), ...]), + (cat, [(name, desc, misc), (name, desc, misc), ...]), + ... + ] + """ + assert model.rowCount() == len(expected) + for i, (expected_title, expected_items) in enumerate(expected): + catidx = model.index(i, 0) + assert model.data(catidx) == expected_title + assert model.rowCount(catidx) == len(expected_items) + for j, (name, desc, misc) in enumerate(expected_items): + assert model.data(model.index(j, 0, catidx)) == name + assert model.data(model.index(j, 1, catidx)) == desc + assert model.data(model.index(j, 2, catidx)) == misc + + +@pytest.mark.parametrize('rowcounts, expected', [ + ([0], 0), + ([1], 1), + ([2], 2), + ([0, 0], 0), + ([0, 0, 0], 0), + ([1, 1], 2), + ([3, 2, 1], 6), + ([0, 2, 0], 2), +]) +def test_count(rowcounts, expected): + model = sqlmodel.SqlCompletionModel() + for i, rowcount in enumerate(rowcounts): + name = 'Foo' + str(i) + table = sql.SqlTable(name, ['a'], primary_key='a') + for rownum in range(rowcount): + table.insert(rownum) + model.new_category(name) + assert model.count() == expected + + +@pytest.mark.parametrize('sort_by, sort_order, data, expected', [ + (None, Qt.AscendingOrder, + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), + + ('a', Qt.AscendingOrder, + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), + + ('a', Qt.DescendingOrder, + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), + + ('b', Qt.AscendingOrder, + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), + + ('b', Qt.DescendingOrder, + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), + + ('c', Qt.AscendingOrder, + [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], + [('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]), + + ('c', Qt.DescendingOrder, + [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], + [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), +]) +def test_sorting(sort_by, sort_order, data, expected): + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + for row in data: + table.insert(*row) + model = sqlmodel.SqlCompletionModel() + model.new_category('Foo', sort_by=sort_by, sort_order=sort_order) + _check_model(model, [('Foo', expected)]) + + +@pytest.mark.parametrize('pattern, filter_cols, before, after', [ + ('foo', [0], + [('A', [('foo', '', ''), ('bar', '', ''), ('aafobbb', '', '')])], + [('A', [('foo', '', '')])]), + + ('foo', [0], + [('A', [('baz', 'bar', 'foo'), ('foo', '', ''), ('bar', 'foo', '')])], + [('A', [('foo', '', '')])]), + + ('foo', [0], + [('A', [('foo', '', ''), ('bar', '', '')]), + ('B', [('foo', '', ''), ('bar', '', '')])], + [('A', [('foo', '', '')]), ('B', [('foo', '', '')])]), + + ('foo', [0], + [('A', [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')])], + [('A', [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')])]), + + ('foo', [0], + [('A', [('foo', '', '')]), ('B', [('bar', '', '')])], + [('A', [('foo', '', '')]), ('B', [])]), + + ('foo', [1], + [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])], + [('A', [('bar', 'foo', '')])]), + + ('foo', [0, 1], + [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])], + [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])]), + + ('foo', [0, 1, 2], + [('A', [('foo', '', ''), ('bar', '', '')])], + [('A', [('foo', '', '')])]), + + ('foo bar', [0], + [('A', [('foo', '', ''), ('bar foo', '', ''), ('xfooyybarz', '', '')])], + [('A', [('xfooyybarz', '', '')])]), + + ('foo%bar', [0], + [('A', [('foo%bar', '', ''), ('foo bar', '', ''), ('foobar', '', '')])], + [('A', [('foo%bar', '', '')])]), + + ('_', [0], + [('A', [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')])], + [('A', [('a_b', '', ''), ('__a', '', '')])]), +]) +def test_set_pattern(pattern, filter_cols, before, after): + """Validate the filtering and sorting results of set_pattern.""" + model = sqlmodel.SqlCompletionModel() + for name, rows in before: + table = sql.SqlTable(name, ['a', 'b', 'c'], primary_key='a') + for row in rows: + table.insert(*row) + model.new_category(name) + model.columns_to_filter = filter_cols + model.set_pattern(pattern) + _check_model(model, after) + + +@pytest.mark.parametrize('data, first, last', [ + ([('A', ['Aa'])], 'Aa', 'Aa'), + ([('A', ['Aa', 'Ba'])], 'Aa', 'Ba'), + ([('A', ['Aa', 'Ab', 'Ac']), ('B', ['Ba', 'Bb']), + ('C', ['Ca'])], 'Aa', 'Ca'), + ([('A', []), ('B', ['Ba'])], 'Ba', 'Ba'), + ([('A', []), ('B', []), ('C', ['Ca'])], 'Ca', 'Ca'), + ([('A', []), ('B', []), ('C', ['Ca', 'Cb'])], 'Ca', 'Cb'), + ([('A', ['Aa']), ('B', [])], 'Aa', 'Aa'), + ([('A', ['Aa']), ('B', []), ('C', [])], 'Aa', 'Aa'), + ([('A', ['Aa']), ('B', []), ('C', ['Ca'])], 'Aa', 'Ca'), + ([('A', []), ('B', [])], None, None), +]) +def test_first_last_item(data, first, last): + """Test that first() and last() return indexes to the first and last items. + + Args: + data: Input to _make_model + first: text of the first item + last: text of the last item + """ + model = sqlmodel.SqlCompletionModel() + for name, rows in data: + table = sql.SqlTable(name, ['a'], primary_key='a') + for row in rows: + table.insert(row) + model.new_category(name) + assert model.data(model.first_item()) == first + assert model.data(model.last_item()) == last From acea0d3c671fb3dc6e8c521710d0c89892b87453 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 5 Feb 2017 19:34:14 -0500 Subject: [PATCH 008/161] Use SQL completion for the open command. Now that history, bookmark, and quickmark storage are SQL-backed, use a sql completion model to serve url completions. --- qutebrowser/completion/models/urlmodel.py | 183 ++-------------------- tests/unit/browser/webkit/test_history.py | 2 +- 2 files changed, 12 insertions(+), 173 deletions(-) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index d8b7f42a2..bd3efd469 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -19,180 +19,19 @@ """Function to return the url completion model for the `open` command.""" -import datetime - -from PyQt5.QtCore import pyqtSlot, Qt - -from qutebrowser.utils import objreg, utils, qtutils, log -from qutebrowser.completion.models import base -from qutebrowser.config import config - -_URL_COLUMN = 0 -_TEXT_COLUMN = 1 -_TIME_COLUMN = 2 -_model = None -_history_cat = None -_quickmark_cat = None -_bookmark_cat = None - - -def _delete_url(completion): - index = completion.currentIndex() - qtutils.ensure_valid(index) - category = index.parent() - index = category.child(index.row(), _URL_COLUMN) - qtutils.ensure_valid(category) - if category.data() == 'Bookmarks': - bookmark_manager = objreg.get('bookmark-manager') - bookmark_manager.delete(index.data()) - elif category.data() == 'Quickmarks': - quickmark_manager = objreg.get('quickmark-manager') - sibling = index.sibling(index.row(), _TEXT_COLUMN) - qtutils.ensure_valid(sibling) - name = sibling.data() - quickmark_manager.delete(name) - - -def _remove_oldest_history(): - """Remove the oldest history entry.""" - _history_cat.removeRow(0) - - -def _add_history_entry(entry): - """Add a new history entry to the completion.""" - _model.new_item(_history_cat, entry.url.toDisplayString(), - entry.title, _fmt_atime(entry.atime), - sort=int(entry.atime), userdata=entry.url) - - max_history = config.get('completion', 'web-history-max-items') - if max_history != -1 and _history_cat.rowCount() > max_history: - _remove_oldest_history() - - -@config.change_filter('completion', 'timestamp-format') -def _reformat_timestamps(): - """Reformat the timestamps if the config option was changed.""" - for i in range(_history_cat.rowCount()): - url_item = _history_cat.child(i, _URL_COLUMN) - atime_item = _history_cat.child(i, _TIME_COLUMN) - atime = url_item.data(base.Role.sort) - atime_item.setText(_fmt_atime(atime)) - - -@pyqtSlot(object) -def _on_history_item_added(entry): - """Slot called when a new history item was added.""" - for i in range(_history_cat.rowCount()): - url_item = _history_cat.child(i, _URL_COLUMN) - atime_item = _history_cat.child(i, _TIME_COLUMN) - title_item = _history_cat.child(i, _TEXT_COLUMN) - if url_item.data(base.Role.userdata) == entry.url: - atime_item.setText(_fmt_atime(entry.atime)) - title_item.setText(entry.title) - url_item.setData(int(entry.atime), base.Role.sort) - break - else: - _add_history_entry(entry) - - -@pyqtSlot() -def _on_history_cleared(): - _history_cat.removeRows(0, _history_cat.rowCount()) - - -def _remove_item(data, category, column): - """Helper function for on_quickmark_removed and on_bookmark_removed. - - Args: - data: The item to search for. - category: The category to search in. - column: The column to use for matching. - """ - for i in range(category.rowCount()): - item = category.child(i, column) - if item.data(Qt.DisplayRole) == data: - category.removeRow(i) - break - - -@pyqtSlot(str) -def _on_quickmark_removed(name): - """Called when a quickmark has been removed by the user. - - Args: - name: The name of the quickmark which has been removed. - """ - _remove_item(name, _quickmark_cat, _TEXT_COLUMN) - - -@pyqtSlot(str) -def _on_bookmark_removed(urltext): - """Called when a bookmark has been removed by the user. - - Args: - urltext: The url of the bookmark which has been removed. - """ - _remove_item(urltext, _bookmark_cat, _URL_COLUMN) - - -def _fmt_atime(atime): - """Format an atime to a human-readable string.""" - fmt = config.get('completion', 'timestamp-format') - if fmt is None: - return '' - try: - dt = datetime.datetime.fromtimestamp(atime) - except (ValueError, OSError, OverflowError): - # Different errors which can occur for too large values... - log.misc.error("Got invalid timestamp {}!".format(atime)) - return '(invalid)' - else: - return dt.strftime(fmt) - - -def _init(): - global _model, _quickmark_cat, _bookmark_cat, _history_cat - _model = base.CompletionModel(column_widths=(40, 50, 10), - dumb_sort=Qt.DescendingOrder, - delete_cur_item=_delete_url, - columns_to_filter=[_URL_COLUMN, - _TEXT_COLUMN]) - _quickmark_cat = _model.new_category("Quickmarks") - _bookmark_cat = _model.new_category("Bookmarks") - _history_cat = _model.new_category("History") - - quickmark_manager = objreg.get('quickmark-manager') - quickmarks = quickmark_manager.marks.items() - for qm_name, qm_url in quickmarks: - _model.new_item(_quickmark_cat, qm_url, qm_name) - quickmark_manager.added.connect( - lambda name, url: _model.new_item(_quickmark_cat, url, name)) - quickmark_manager.removed.connect(_on_quickmark_removed) - - bookmark_manager = objreg.get('bookmark-manager') - bookmarks = bookmark_manager.marks.items() - for bm_url, bm_title in bookmarks: - _model.new_item(_bookmark_cat, bm_url, bm_title) - bookmark_manager.added.connect( - lambda name, url: _model.new_item(_bookmark_cat, url, name)) - bookmark_manager.removed.connect(_on_bookmark_removed) - - history = objreg.get('web-history') - max_history = config.get('completion', 'web-history-max-items') - for entry in utils.newest_slice(history, max_history): - if not entry.redirect: - _add_history_entry(entry) - history.add_completion_item.connect(_on_history_item_added) - history.cleared.connect(_on_history_cleared) - - objreg.get('config').changed.connect(_reformat_timestamps) - +from qutebrowser.completion.models import sqlmodel def url(): - """A _model which combines bookmarks, quickmarks and web history URLs. + """A model which combines bookmarks, quickmarks and web history URLs. Used for the `open` command. """ - if not _model: - _init() - return _model + urlcol = 0 + textcol = 1 + + model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), + columns_to_filter=[urlcol, textcol]) + model.new_category('History') + model.new_category('Quickmarks') + model.new_category('Bookmarks') + return model diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 6ad461692..de7c27a77 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -27,7 +27,7 @@ from hypothesis import strategies from PyQt5.QtCore import QUrl from qutebrowser.browser import history -from qutebrowser.utils import objreg, urlutils +from qutebrowser.utils import objreg, urlutils, usertypes from qutebrowser.misc import sql From 3005374ada236d8b126e2194881663401edb654c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 8 Feb 2017 17:21:27 -0500 Subject: [PATCH 009/161] Simplify sorting logic in sortfilter. For URL completion, time-based sorting is handled by the SQL model. All the other models use simple alphabetical sorting. This allowed cleaning up some logic in the sortfilter, removing DUMB_SORT, and removing the completion.Role.sort. This also removes the userdata completion field as it was only used in url completion and is no longer necessary with the SQL model. --- qutebrowser/completion/models/base.py | 26 ++-------- qutebrowser/completion/models/configmodel.py | 4 +- qutebrowser/completion/models/miscmodels.py | 1 - qutebrowser/completion/models/sortfilter.py | 28 ++-------- tests/unit/completion/test_completer.py | 1 - tests/unit/completion/test_sortfilter.py | 54 ++++---------------- 6 files changed, 19 insertions(+), 95 deletions(-) diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py index 43f3a1b48..1d3ed2be4 100644 --- a/qutebrowser/completion/models/base.py +++ b/qutebrowser/completion/models/base.py @@ -29,10 +29,6 @@ from PyQt5.QtGui import QStandardItemModel, QStandardItem from qutebrowser.utils import usertypes -Role = usertypes.enum('Role', ['sort', 'userdata'], start=Qt.UserRole, - is_int=True) - - class CompletionModel(QStandardItemModel): """A simple QStandardItemModel adopted for completions. @@ -42,37 +38,30 @@ class CompletionModel(QStandardItemModel): Attributes: column_widths: The width percentages of the columns used in the - completion view. - dumb_sort: the dumb sorting used by the model """ - def __init__(self, dumb_sort=None, column_widths=(30, 70, 0), - columns_to_filter=None, delete_cur_item=None, parent=None): + def __init__(self, column_widths=(30, 70, 0), columns_to_filter=None, + delete_cur_item=None, parent=None): super().__init__(parent) self.setColumnCount(3) self.columns_to_filter = columns_to_filter or [0] - self.dumb_sort = dumb_sort self.column_widths = column_widths self.delete_cur_item = delete_cur_item - def new_category(self, name, sort=None): + def new_category(self, name): """Add a new category to the model. Args: name: The name of the category to add. - sort: The value to use for the sort role. Return: The created QStandardItem. """ cat = QStandardItem(name) - if sort is not None: - cat.setData(sort, Role.sort) self.appendRow(cat) return cat - def new_item(self, cat, name, desc='', misc=None, sort=None, - userdata=None): + def new_item(self, cat, name, desc='', misc=None): """Add a new item to a category. Args: @@ -80,8 +69,6 @@ class CompletionModel(QStandardItemModel): name: The name of the item. desc: The description of the item. misc: Misc text to display. - sort: Data for the sort role (int). - userdata: User data to be added for the first column. Return: A (nameitem, descitem, miscitem) tuple. @@ -98,11 +85,6 @@ class CompletionModel(QStandardItemModel): miscitem = QStandardItem(misc) cat.appendRow([nameitem, descitem, miscitem]) - if sort is not None: - nameitem.setData(sort, Role.sort) - if userdata is not None: - nameitem.setData(userdata, Role.userdata) - return nameitem, descitem, miscitem def flags(self, index): """Return the item flags for index. diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index 5ed2a47d0..406d8d572 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -67,7 +67,7 @@ def value(sectname, optname): optname: The name of the config option this model shows. """ model = base.CompletionModel(column_widths=(20, 70, 10)) - cur_cat = model.new_category("Current/Default", sort=0) + cur_cat = model.new_category("Current/Default") try: val = config.get(sectname, optname, raw=True) or '""' except (configexc.NoSectionError, configexc.NoOptionError): @@ -85,7 +85,7 @@ def value(sectname, optname): # Different type for each value (KeyValue) vals = configdata.DATA[sectname][optname].typ.complete() if vals is not None: - cat = model.new_category("Completions", sort=1) + cat = model.new_category("Completions") for (val, desc) in vals: model.new_item(cat, val, desc) return model diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index bcbb94177..cb87f9d7e 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -119,7 +119,6 @@ def buffer(): model = base.CompletionModel( column_widths=(6, 40, 54), - dumb_sort=Qt.DescendingOrder, delete_cur_item=delete_buffer, columns_to_filter=[idx_column, url_column, text_column]) diff --git a/qutebrowser/completion/models/sortfilter.py b/qutebrowser/completion/models/sortfilter.py index 92dd1b2a0..e5d42607c 100644 --- a/qutebrowser/completion/models/sortfilter.py +++ b/qutebrowser/completion/models/sortfilter.py @@ -33,14 +33,13 @@ from qutebrowser.completion.models import base as completion class CompletionFilterModel(QSortFilterProxyModel): - """Subclass of QSortFilterProxyModel with custom sorting/filtering. + """Subclass of QSortFilterProxyModel with custom filtering. Attributes: pattern: The pattern to filter with. srcmodel: The current source model. Kept as attribute because calling `sourceModel` takes quite a long time for some reason. - _sort_order: The order to use for sorting if using dumb_sort. """ def __init__(self, source, parent=None): @@ -49,21 +48,12 @@ class CompletionFilterModel(QSortFilterProxyModel): self.srcmodel = source self.pattern = '' self.pattern_re = None - - dumb_sort = self.srcmodel.dumb_sort - if dumb_sort is None: - # pylint: disable=invalid-name - self.lessThan = self.intelligentLessThan - self._sort_order = Qt.AscendingOrder - else: - self.setSortRole(completion.Role.sort) - self._sort_order = dumb_sort + self.lessThan = self.intelligentLessThan + #self._sort_order = self.srcmodel.sort_order or Qt.AscendingOrder def set_pattern(self, val): """Setter for pattern. - Invalidates the filter and re-sorts the model. - Args: val: The value to set. """ @@ -163,12 +153,6 @@ class CompletionFilterModel(QSortFilterProxyModel): qtutils.ensure_valid(lindex) qtutils.ensure_valid(rindex) - left_sort = self.srcmodel.data(lindex, role=completion.Role.sort) - right_sort = self.srcmodel.data(rindex, role=completion.Role.sort) - - if left_sort is not None and right_sort is not None: - return left_sort < right_sort - left = self.srcmodel.data(lindex) right = self.srcmodel.data(rindex) @@ -183,9 +167,3 @@ class CompletionFilterModel(QSortFilterProxyModel): return False else: return left < right - - def sort(self, column, order=None): - """Extend sort to respect self._sort_order if no order was given.""" - if order is None: - order = self._sort_order - super().sort(column, order) diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index 48f619104..34c9a2d19 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -37,7 +37,6 @@ class FakeCompletionModel(QStandardItemModel): super().__init__(parent) self.kind = kind self.pos_args = [*pos_args] - self.dumb_sort = None class CompletionWidgetStub(QObject): diff --git a/tests/unit/completion/test_sortfilter.py b/tests/unit/completion/test_sortfilter.py index c972a5ec9..7d84e9489 100644 --- a/tests/unit/completion/test_sortfilter.py +++ b/tests/unit/completion/test_sortfilter.py @@ -151,80 +151,46 @@ def test_count(tree, expected): assert filter_model.count() == expected -@pytest.mark.parametrize('pattern, dumb_sort, filter_cols, before, after', [ - ('foo', None, [0], +@pytest.mark.parametrize('pattern, filter_cols, before, after', [ + ('foo', [0], [[('foo', '', ''), ('bar', '', '')]], [[('foo', '', '')]]), - ('foo', None, [0], + ('foo', [0], [[('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')]], [[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]]), - ('foo', None, [0], + ('foo', [0], [[('foo', '', '')], [('bar', '', '')]], [[('foo', '', '')], []]), # prefer foobar as it starts with the pattern - ('foo', None, [0], + ('foo', [0], [[('barfoo', '', ''), ('foobar', '', '')]], [[('foobar', '', ''), ('barfoo', '', '')]]), # however, don't rearrange categories - ('foo', None, [0], + ('foo', [0], [[('barfoo', '', '')], [('foobar', '', '')]], [[('barfoo', '', '')], [('foobar', '', '')]]), - ('foo', None, [1], + ('foo', [1], [[('foo', 'bar', ''), ('bar', 'foo', '')]], [[('bar', 'foo', '')]]), - ('foo', None, [0, 1], + ('foo', [0, 1], [[('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')]], [[('foo', 'bar', ''), ('bar', 'foo', '')]]), - ('foo', None, [0, 1, 2], + ('foo', [0, 1, 2], [[('foo', '', ''), ('bar', '')]], [[('foo', '', '')]]), - - # the fourth column is the sort role, which overrides data-based sorting - ('', None, [0], - [[('two', '', '', 2), ('one', '', '', 1), ('three', '', '', 3)]], - [[('one', '', ''), ('two', '', ''), ('three', '', '')]]), - - ('', Qt.AscendingOrder, [0], - [[('two', '', '', 2), ('one', '', '', 1), ('three', '', '', 3)]], - [[('one', '', ''), ('two', '', ''), ('three', '', '')]]), - - ('', Qt.DescendingOrder, [0], - [[('two', '', '', 2), ('one', '', '', 1), ('three', '', '', 3)]], - [[('three', '', ''), ('two', '', ''), ('one', '', '')]]), ]) -def test_set_pattern(pattern, dumb_sort, filter_cols, before, after): +def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" model = _create_model(before) - model.dumb_sort = dumb_sort model.columns_to_filter = filter_cols filter_model = sortfilter.CompletionFilterModel(model) filter_model.set_pattern(pattern) actual = _extract_model_data(filter_model) assert actual == after - - -def test_sort(): - """Ensure that a sort argument passed to sort overrides DUMB_SORT. - - While test_set_pattern above covers most of the sorting logic, this - particular case is easier to test separately. - """ - model = _create_model([[('B', '', '', 1), - ('C', '', '', 2), - ('A', '', '', 0)]]) - filter_model = sortfilter.CompletionFilterModel(model) - - filter_model.sort(0, Qt.AscendingOrder) - actual = _extract_model_data(filter_model) - assert actual == [[('A', '', ''), ('B', '', ''), ('C', '', '')]] - - filter_model.sort(0, Qt.DescendingOrder) - actual = _extract_model_data(filter_model) - assert actual == [[('C', '', ''), ('B', '', ''), ('A', '', '')]] From 93f89849871c64e21536d2b47355c89e55c37eaa Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 27 Nov 2016 16:52:03 -0500 Subject: [PATCH 010/161] Install pyqt5.qtsql bindings for debian CI. SQL is included in the Archlinux pyqt5 package, but not in Debian. We need this so the debian-based CI builds will pass with the new sql-based completion implementation. --- scripts/dev/ci/travis_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index 0ada134c1..aa4c1c6c8 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -102,7 +102,7 @@ elif [[ $TRAVIS_OS_NAME == osx ]]; then exit 0 fi -pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit" +pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit python3-pyqt5.qtsql" pip_install pip pip_install -r misc/requirements/requirements-tox.txt From d4f2a70f83a2ee6fbbb580a81fc22094e0f354fe Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 08:36:50 -0500 Subject: [PATCH 011/161] Slightly simplify CompletionModel.new_item. There was no need to have a branch based on whether the misc value was None or not. --- qutebrowser/completion/models/base.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py index 1d3ed2be4..5e2c78c5a 100644 --- a/qutebrowser/completion/models/base.py +++ b/qutebrowser/completion/models/base.py @@ -79,10 +79,7 @@ class CompletionModel(QStandardItemModel): nameitem = QStandardItem(name) descitem = QStandardItem(desc) - if misc is None: - miscitem = QStandardItem() - else: - miscitem = QStandardItem(misc) + miscitem = QStandardItem(misc) cat.appendRow([nameitem, descitem, miscitem]) From a774647c260b1a9d920c8699a60aa0a992b814bf Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 08:39:24 -0500 Subject: [PATCH 012/161] Get test_models mostly working again. - Adjust _check_completions to work for CompletionModel and SqlCompletionModel - Move sql initialization into a reusable fixture - Remove the bookmark/quickmark/history stubs, as they're now handled by sql - Disable quickmark/bookmark model tests until their completion is ported to sql. - Disable urlmodel tests for features that have to be implemented in SQL: - LIMIT (for history-max-items) - Configurable column order (for quickmarks) - Configurable formatting (for timestamp-format --- tests/helpers/fixtures.py | 37 ++----- tests/unit/browser/webkit/test_history.py | 5 +- tests/unit/completion/test_models.py | 114 ++++++++++++---------- tests/unit/completion/test_sqlmodel.py | 6 +- tests/unit/misc/test_sql.py | 6 +- 5 files changed, 76 insertions(+), 92 deletions(-) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 3af7195f5..7fc05a578 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -41,7 +41,7 @@ import helpers.stubs as stubsmod from qutebrowser.config import config from qutebrowser.utils import objreg, standarddir from qutebrowser.browser.webkit import cookies -from qutebrowser.misc import savemanager +from qutebrowser.misc import savemanager, sql from qutebrowser.keyinput import modeman from PyQt5.QtCore import PYQT_VERSION, pyqtSignal, QEvent, QSize, Qt, QObject @@ -239,33 +239,6 @@ def host_blocker_stub(stubs): objreg.delete('host-blocker') -@pytest.fixture -def quickmark_manager_stub(stubs): - """Fixture which provides a fake quickmark manager object.""" - stub = stubs.QuickmarkManagerStub() - objreg.register('quickmark-manager', stub) - yield stub - objreg.delete('quickmark-manager') - - -@pytest.fixture -def bookmark_manager_stub(stubs): - """Fixture which provides a fake bookmark manager object.""" - stub = stubs.BookmarkManagerStub() - objreg.register('bookmark-manager', stub) - yield stub - objreg.delete('bookmark-manager') - - -@pytest.fixture -def web_history_stub(stubs): - """Fixture which provides a fake web-history object.""" - stub = stubs.WebHistoryStub() - objreg.register('web-history', stub) - yield stub - objreg.delete('web-history') - - @pytest.fixture def session_manager_stub(stubs): """Fixture which provides a fake web-history object.""" @@ -482,3 +455,11 @@ def short_tmpdir(): """A short temporary directory for a XDG_RUNTIME_DIR.""" with tempfile.TemporaryDirectory() as tdir: yield py.path.local(tdir) # pylint: disable=no-member + + +@pytest.fixture() +def init_sql(): + """Initialize the SQL module, and shut it down after the test.""" + sql.init() + yield + sql.close() diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index de7c27a77..ff6b78ae2 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -32,12 +32,9 @@ from qutebrowser.misc import sql @pytest.fixture(autouse=True) -def prerequisites(config_stub, fake_save_manager): +def prerequisites(config_stub, fake_save_manager, init_sql): """Make sure everything is ready to initialize a WebHistory.""" config_stub.data = {'general': {'private-browsing': False}} - sql.init() - yield - sql.close() @pytest.fixture() diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 9870bb4db..a41bc6f02 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -30,6 +30,7 @@ from qutebrowser.completion.models import (miscmodels, urlmodel, configmodel, sortfilter) from qutebrowser.browser import history from qutebrowser.config import sections, value +from qutebrowser.misc import sql def _check_completions(model, expected): @@ -45,17 +46,16 @@ def _check_completions(model, expected): """ assert model.rowCount() == len(expected) for i in range(0, model.rowCount()): - actual_cat = model.item(i) - catname = actual_cat.text() + catidx = model.index(i, 0) + catname = model.data(catidx) assert catname in expected expected_cat = expected[catname] - assert actual_cat.rowCount() == len(expected_cat) - for j in range(0, actual_cat.rowCount()): - name = actual_cat.child(j, 0) - desc = actual_cat.child(j, 1) - misc = actual_cat.child(j, 2) - actual_item = (name.text(), desc.text(), misc.text()) - assert actual_item in expected_cat + assert model.rowCount(catidx) == len(expected_cat) + for j in range(model.rowCount(catidx)): + name = model.data(model.index(j, 0, parent=catidx)) + desc = model.data(model.index(j, 1, parent=catidx)) + misc = model.data(model.index(j, 2, parent=catidx)) + assert (name, desc, misc) in expected_cat # sanity-check the column_widths assert len(model.column_widths) == 3 assert sum(model.column_widths) == 100 @@ -133,42 +133,34 @@ def _mock_view_index(model, category_idx, child_idx, qtbot): @pytest.fixture -def quickmarks(quickmark_manager_stub): - """Pre-populate the quickmark-manager stub with some quickmarks.""" - quickmark_manager_stub.marks = collections.OrderedDict([ - ('aw', 'https://wiki.archlinux.org'), - ('ddg', 'https://duckduckgo.com'), - ('wiki', 'https://wikipedia.org'), - ]) - return quickmark_manager_stub +def quickmarks(init_sql): + """Pre-populate the quickmark database.""" + table = sql.SqlTable('Quickmarks', ['name', 'url'], primary_key='name') + table.insert('aw', 'https://wiki.archlinux.org') + table.insert('ddg', 'https://duckduckgo.com') + table.insert('wiki', 'https://wikipedia.org') @pytest.fixture -def bookmarks(bookmark_manager_stub): - """Pre-populate the bookmark-manager stub with some quickmarks.""" - bookmark_manager_stub.marks = collections.OrderedDict([ - ('https://github.com', 'GitHub'), - ('https://python.org', 'Welcome to Python.org'), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser'), - ]) - return bookmark_manager_stub +def bookmarks(init_sql): + """Pre-populate the bookmark database.""" + table = sql.SqlTable('Bookmarks', ['url', 'title'], primary_key='url') + table.insert('https://github.com', 'GitHub') + table.insert('https://python.org', 'Welcome to Python.org') + table.insert('http://qutebrowser.org', 'qutebrowser | qutebrowser') @pytest.fixture -def web_history(stubs, web_history_stub): - """Pre-populate the web-history stub with some history entries.""" - web_history_stub.history_dict = collections.OrderedDict([ - ('http://qutebrowser.org', history.Entry( - datetime(2015, 9, 5).timestamp(), - QUrl('http://qutebrowser.org'), 'qutebrowser | qutebrowser')), - ('https://python.org', history.Entry( - datetime(2016, 3, 8).timestamp(), - QUrl('https://python.org'), 'Welcome to Python.org')), - ('https://github.com', history.Entry( - datetime(2016, 5, 1).timestamp(), - QUrl('https://github.com'), 'GitHub')), - ]) - return web_history_stub +def web_history(stubs, init_sql): + """Pre-populate the web-history database.""" + table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect'], + primary_key='url') + table.insert('http://qutebrowser.org', 'qutebrowser', + datetime(2015, 9, 5).timestamp(), False) + table.insert('https://python.org', 'Welcome to Python.org', + datetime(2016, 3, 8).timestamp(), False) + table.insert('https://github.com', 'https://github.com', + datetime(2016, 5, 1).timestamp(), False) def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, @@ -237,6 +229,7 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): }) +@pytest.mark.skip def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() @@ -252,6 +245,7 @@ def test_quickmark_completion(qtmodeltester, quickmarks): }) +@pytest.mark.skip def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() @@ -273,33 +267,52 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, Verify that: - quickmarks, bookmarks, and urls are included - - no more than 'web-history-max-items' history entries are included + - no more than 'web-history-max-items' items are included (TODO) - the most recent entries are included """ - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - 'web-history-max-items': 2} + # TODO: time formatting and item limiting + #config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', + # 'web-history-max-items': 2} model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { + # TODO: rearrange columns so address comes first + #"Quickmarks": [ + # ('https://wiki.archlinux.org', 'aw', None), + # ('https://duckduckgo.com', 'ddg', None), + # ('https://wikipedia.org', 'wiki', None), + #], "Quickmarks": [ - ('https://wiki.archlinux.org', 'aw', ''), - ('https://duckduckgo.com', 'ddg', ''), - ('https://wikipedia.org', 'wiki', ''), + ('aw', 'https://wiki.archlinux.org', None), + ('ddg', 'https://duckduckgo.com', None), + ('wiki', 'https://wikipedia.org', None), ], "Bookmarks": [ - ('https://github.com', 'GitHub', ''), - ('https://python.org', 'Welcome to Python.org', ''), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', ''), + ('https://github.com', 'GitHub', None), + ('https://python.org', 'Welcome to Python.org', None), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ], + # TODO: time formatting and item limiting + #"History": [ + # ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + # ('https://github.com', 'GitHub', '2016-05-01'), + # ('http://qutebrowser.org', 'qutebrowser | qutebrowser', + # '2015-09-05', False) + #], "History": [ - ('https://python.org', 'Welcome to Python.org', '2016-03-08'), - ('https://github.com', 'GitHub', '2016-05-01'), + ('http://qutebrowser.org', 'qutebrowser', + datetime(2015, 9, 5).timestamp()), + ('https://python.org', 'Welcome to Python.org', + datetime(2016, 3, 8).timestamp()), + ('https://github.com', 'https://github.com', + datetime(2016, 5, 1).timestamp()), ], }) +@pytest.mark.skip def test_url_completion_delete_bookmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): @@ -318,6 +331,7 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, assert 'http://qutebrowser.org' in bookmarks.marks +@pytest.mark.skip def test_url_completion_delete_quickmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 6b8f01f4d..49cb216dd 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -26,11 +26,7 @@ from qutebrowser.misc import sql from qutebrowser.completion.models import sqlmodel -@pytest.fixture(autouse=True) -def init(): - sql.init() - yield - sql.close() +pytestmark = pytest.mark.usefixtures('init_sql') def _check_model(model, expected): diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 175f2f97e..d11ed7903 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -23,11 +23,7 @@ import pytest from qutebrowser.misc import sql -@pytest.fixture(autouse=True) -def init(): - sql.init() - yield - sql.close() +pytestmark = pytest.mark.usefixtures('init_sql') def test_init(): From c3155afc21137eb074f25cc032d86a8cc731e8bb Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 08:53:50 -0500 Subject: [PATCH 013/161] Rethrow KeyError as DoesNotExistError in urlmarks. From @TheCompiler: To expand on this: I think it's fine to use KeyError on a lower level, i.e. with the SqlTable object with a dict-like interface. However, on this higher level, I think it makes sense to re-raise them as more specific exceptions. --- qutebrowser/browser/commands.py | 2 +- qutebrowser/browser/urlmarks.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 9d19914f8..e5ab0af21 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1261,7 +1261,7 @@ class CommandDispatcher: url = self._current_url() try: quickmark_manager.delete_by_qurl(url) - except KeyError as e: + except urlmarks.DoesNotExistError as e: raise cmdexc.CommandError(str(e)) else: try: diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index db5e14153..ca71b3f0d 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -53,6 +53,13 @@ class InvalidUrlError(Error): pass +class DoesNotExistError(Error): + + """Exception emitted when a given URL does not exist.""" + + pass + + class AlreadyExistsError(Error): """Exception emitted when a given URL does already exist.""" @@ -179,12 +186,14 @@ class QuickmarkManager(UrlMarkManager): try: self.delete(urlstr, field='url') except KeyError: - raise KeyError("Quickmark for '{}' not found!".format(urlstr)) + raise DoesNotExistError("Quickmark for '{}' not found!" + .format(urlstr)) def get(self, name): """Get the URL of the quickmark named name as a QUrl.""" if name not in self: - raise KeyError("Quickmark '{}' does not exist!".format(name)) + raise DoesNotExistError("Quickmark '{}' does not exist!" + .format(name)) urlstr = self[name] try: url = urlutils.fuzzy_url(urlstr, do_search=False) From b381148e06c7e62647c2b8e0ecd7b09e590bf471 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 18:03:00 -0500 Subject: [PATCH 014/161] Unittest CompletionView.completion_item_del. There were no unit tests for this and the various ways it can fail, and I'm about to screw with it a bit. --- .../unit/completion/test_completionwidget.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 88f4aed20..2c8d96713 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -26,6 +26,7 @@ from PyQt5.QtGui import QStandardItem, QColor from qutebrowser.completion import completionwidget from qutebrowser.completion.models import base, sortfilter +from qutebrowser.commands import cmdexc @pytest.fixture @@ -69,6 +70,16 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry, return view +@pytest.fixture +def simplemodel(completionview): + """A filter model wrapped around a completion model with one item.""" + model = base.CompletionModel() + cat = QStandardItem() + cat.appendRow(QStandardItem('foo')) + model.appendRow(cat) + return sortfilter.CompletionFilterModel(model, parent=completionview) + + def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" model = base.CompletionModel() @@ -218,3 +229,29 @@ def test_completion_show(show, rows, quick_complete, completionview, completionview.set_model(None) completionview.completion_item_focus('next') assert not completionview.isVisible() + + +def test_completion_item_del(completionview, simplemodel): + """Test that completion_item_del invokes delete_cur_item in the model.""" + simplemodel.srcmodel.delete_cur_item = unittest.mock.Mock() + completionview.set_model(simplemodel) + completionview.completion_item_focus('next') + completionview.completion_item_del() + assert simplemodel.srcmodel.delete_cur_item.called + + +def test_completion_item_del_no_selection(completionview, simplemodel): + """Test that completion_item_del with no selected index.""" + simplemodel.srcmodel.delete_cur_item = unittest.mock.Mock() + completionview.set_model(simplemodel) + with pytest.raises(cmdexc.CommandError): + completionview.completion_item_del() + assert not simplemodel.srcmodel.delete_cur_item.called + + +def test_completion_item_del_no_func(completionview, simplemodel): + """Test completion_item_del with no delete_cur_item in the model.""" + completionview.set_model(simplemodel) + completionview.completion_item_focus('next') + with pytest.raises(cmdexc.CommandError): + completionview.completion_item_del() From 5bd047b70b2e2f48d5191ef76505bd3154cbed83 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 18:03:49 -0500 Subject: [PATCH 015/161] Fix CompletionView.completion_item_del for new API. The new function based completion models work a little differently so the view needed slightly different error handling. --- qutebrowser/completion/completionwidget.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index f12027340..3d2aef27e 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -368,7 +368,7 @@ class CompletionView(QTreeView): """Delete the current completion item.""" if not self.currentIndex().isValid(): raise cmdexc.CommandError("No item selected!") - try: - self.model().srcmodel.delete_cur_item(self) - except NotImplementedError: + if self.model().srcmodel.delete_cur_item is None: raise cmdexc.CommandError("Cannot delete this item.") + else: + self.model().srcmodel.delete_cur_item(self) From 839d49a8acb544bf4952454b03c028dfbe44409e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 18:14:46 -0500 Subject: [PATCH 016/161] Fix up pylint/flake8 for completion revamp. --- qutebrowser/browser/history.py | 3 +-- qutebrowser/browser/urlmarks.py | 3 +-- qutebrowser/completion/models/base.py | 2 -- qutebrowser/completion/models/miscmodels.py | 2 -- qutebrowser/completion/models/sortfilter.py | 7 ++----- qutebrowser/completion/models/sqlmodel.py | 4 ++-- qutebrowser/completion/models/urlmodel.py | 1 + qutebrowser/config/parsers/keyconf.py | 2 +- tests/unit/browser/webkit/test_history.py | 1 - tests/unit/completion/test_models.py | 20 +++++++++----------- tests/unit/completion/test_sortfilter.py | 2 -- tests/unit/misc/test_sql.py | 2 +- 12 files changed, 18 insertions(+), 31 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index a1fc7dfa2..955de21f2 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -20,9 +20,8 @@ """Simple history which gets written to disk.""" import time -import collections -from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject +from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl from qutebrowser.commands import cmdutils from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils, diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index ca71b3f0d..6d692da0a 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -29,9 +29,8 @@ import os import html import os.path import functools -import collections -from PyQt5.QtCore import QUrl, QObject +from PyQt5.QtCore import QUrl from qutebrowser.utils import (message, usertypes, qtutils, urlutils, standarddir, objreg, log) diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py index 5e2c78c5a..112c6ed00 100644 --- a/qutebrowser/completion/models/base.py +++ b/qutebrowser/completion/models/base.py @@ -26,8 +26,6 @@ Module attributes: from PyQt5.QtCore import Qt from PyQt5.QtGui import QStandardItemModel, QStandardItem -from qutebrowser.utils import usertypes - class CompletionModel(QStandardItemModel): diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index cb87f9d7e..5cb955216 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -19,8 +19,6 @@ """Functions that return miscellaneous completion models.""" -from PyQt5.QtCore import Qt - from qutebrowser.config import config, configdata from qutebrowser.utils import objreg, log, qtutils from qutebrowser.commands import cmdutils diff --git a/qutebrowser/completion/models/sortfilter.py b/qutebrowser/completion/models/sortfilter.py index e5d42607c..92d9aeaf1 100644 --- a/qutebrowser/completion/models/sortfilter.py +++ b/qutebrowser/completion/models/sortfilter.py @@ -25,10 +25,9 @@ Contains: import re -from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex, Qt +from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex from qutebrowser.utils import log, qtutils, debug -from qutebrowser.completion.models import base as completion class CompletionFilterModel(QSortFilterProxyModel): @@ -48,8 +47,6 @@ class CompletionFilterModel(QSortFilterProxyModel): self.srcmodel = source self.pattern = '' self.pattern_re = None - self.lessThan = self.intelligentLessThan - #self._sort_order = self.srcmodel.sort_order or Qt.AscendingOrder def set_pattern(self, val): """Setter for pattern. @@ -137,7 +134,7 @@ class CompletionFilterModel(QSortFilterProxyModel): return True return False - def intelligentLessThan(self, lindex, rindex): + def lessThan(self, lindex, rindex): """Custom sorting implementation. Prefers all items which start with self.pattern. Other than that, uses diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index a5a9bc99a..2f0a5059b 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -22,9 +22,9 @@ import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel -from PyQt5.QtSql import QSqlTableModel, QSqlDatabase, QSqlQuery +from PyQt5.QtSql import QSqlTableModel, QSqlDatabase -from qutebrowser.utils import usertypes, log +from qutebrowser.utils import log class SqlCompletionModel(QAbstractItemModel): diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index bd3efd469..1390fff21 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -21,6 +21,7 @@ from qutebrowser.completion.models import sqlmodel + def url(): """A model which combines bookmarks, quickmarks and web history URLs. diff --git a/qutebrowser/config/parsers/keyconf.py b/qutebrowser/config/parsers/keyconf.py index e4adb8676..5a780a786 100644 --- a/qutebrowser/config/parsers/keyconf.py +++ b/qutebrowser/config/parsers/keyconf.py @@ -27,7 +27,7 @@ from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.config import configdata, textwrapper from qutebrowser.commands import cmdutils, cmdexc -from qutebrowser.utils import log, utils, qtutils, message +from qutebrowser.utils import log, utils, qtutils, message, usertypes from qutebrowser.completion.models import miscmodels diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index ff6b78ae2..844c20fb1 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -28,7 +28,6 @@ from PyQt5.QtCore import QUrl from qutebrowser.browser import history from qutebrowser.utils import objreg, urlutils, usertypes -from qutebrowser.misc import sql @pytest.fixture(autouse=True) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index a41bc6f02..2e687a344 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -26,9 +26,8 @@ import pytest from PyQt5.QtCore import QUrl from PyQt5.QtWidgets import QTreeView -from qutebrowser.completion.models import (miscmodels, urlmodel, configmodel, - sortfilter) from qutebrowser.browser import history +from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value from qutebrowser.misc import sql @@ -544,14 +543,13 @@ def test_url_completion_benchmark(benchmark, config_stub, for e in entries[0:1000]) def bench(): - model = urlmodel.UrlCompletionModel() - filtermodel = sortfilter.CompletionFilterModel(model) - filtermodel.set_pattern('') - filtermodel.set_pattern('e') - filtermodel.set_pattern('ex') - filtermodel.set_pattern('ex ') - filtermodel.set_pattern('ex 1') - filtermodel.set_pattern('ex 12') - filtermodel.set_pattern('ex 123') + model = urlmodel.url() + model.set_pattern('') + model.set_pattern('e') + model.set_pattern('ex') + model.set_pattern('ex ') + model.set_pattern('ex 1') + model.set_pattern('ex 12') + model.set_pattern('ex 123') benchmark(bench) diff --git a/tests/unit/completion/test_sortfilter.py b/tests/unit/completion/test_sortfilter.py index 7d84e9489..92d644810 100644 --- a/tests/unit/completion/test_sortfilter.py +++ b/tests/unit/completion/test_sortfilter.py @@ -21,8 +21,6 @@ import pytest -from PyQt5.QtCore import Qt - from qutebrowser.completion.models import base, sortfilter diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index d11ed7903..889e28a19 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -74,7 +74,7 @@ def test_delete(qtbot): assert list(table) == [('one', 1, False), ('nine', 9, False)] with qtbot.waitSignal(table.changed): table.delete(False, field='lucky') - assert not list(table) == [('thirteen', 13, True)] + assert not list(table) def test_len(): From 52d7d1df0c9ff754dcf7e1214fe7de6f8fd4271e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 18:24:09 -0500 Subject: [PATCH 017/161] Use SQL completer for quickmarks/bookmarks. --- qutebrowser/completion/models/miscmodels.py | 14 +++++--------- tests/unit/completion/test_models.py | 14 ++++++-------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 5cb955216..8bb234521 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -22,7 +22,7 @@ from qutebrowser.config import config, configdata from qutebrowser.utils import objreg, log, qtutils from qutebrowser.commands import cmdutils -from qutebrowser.completion.models import base +from qutebrowser.completion.models import base, sqlmodel def command(): @@ -64,20 +64,16 @@ def helptopic(): def quickmark(): """A CompletionModel filled with all quickmarks.""" model = base.CompletionModel() - cat = model.new_category("Quickmarks") - quickmarks = objreg.get('quickmark-manager').marks.items() - for qm_name, qm_url in quickmarks: - model.new_item(cat, qm_name, qm_url) + model = sqlmodel.SqlCompletionModel(column_widths=(30, 70, 0)) + model.new_category('Quickmarks') return model def bookmark(): """A CompletionModel filled with all bookmarks.""" model = base.CompletionModel() - cat = model.new_category("Bookmarks") - bookmarks = objreg.get('bookmark-manager').marks.items() - for bm_url, bm_title in bookmarks: - model.new_item(cat, bm_url, bm_title) + model = sqlmodel.SqlCompletionModel(column_widths=(30, 70, 0)) + model.new_category('Bookmarks') return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 2e687a344..e1366325c 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -228,7 +228,6 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): }) -@pytest.mark.skip def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() @@ -237,14 +236,13 @@ def test_quickmark_completion(qtmodeltester, quickmarks): _check_completions(model, { "Quickmarks": [ - ('aw', 'https://wiki.archlinux.org', ''), - ('ddg', 'https://duckduckgo.com', ''), - ('wiki', 'https://wikipedia.org', ''), + ('aw', 'https://wiki.archlinux.org', None), + ('ddg', 'https://duckduckgo.com', None), + ('wiki', 'https://wikipedia.org', None), ] }) -@pytest.mark.skip def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() @@ -253,9 +251,9 @@ def test_bookmark_completion(qtmodeltester, bookmarks): _check_completions(model, { "Bookmarks": [ - ('https://github.com', 'GitHub', ''), - ('https://python.org', 'Welcome to Python.org', ''), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', ''), + ('https://github.com', 'GitHub', None), + ('https://python.org', 'Welcome to Python.org', None), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ] }) From b70d5ba901de09aee744993913a58e910d884936 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 08:46:31 -0500 Subject: [PATCH 018/161] Use QSqlQueryModel instead of QSqlTableModel. This allows setting the query as a QSqlQuery instead of a string, which allows: - Escaping quotes - Using LIMIT (needed for history-max-items) - Using ORDER BY (needed for sorting history) - SELECTing columns (needed for quickmark completion) - Creating a custom select (needed for history timestamp formatting) --- qutebrowser/completion/models/sqlmodel.py | 58 ++++++++++++++++------- qutebrowser/misc/sql.py | 18 +++---- tests/unit/completion/test_sqlmodel.py | 15 +++++- 3 files changed, 62 insertions(+), 29 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 2f0a5059b..40b2e27ca 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -22,9 +22,39 @@ import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel -from PyQt5.QtSql import QSqlTableModel, QSqlDatabase +from PyQt5.QtSql import QSqlQuery, QSqlQueryModel, QSqlDatabase from qutebrowser.utils import log +from qutebrowser.misc import sql + + +class SqlCompletionCategory(QSqlQueryModel): + def __init__(self, name, sort_by, sort_order, limit, columns_to_filter, + parent=None): + super().__init__(parent=parent) + self.tablename = name + + query = sql.run_query('select * from {} limit 1'.format(name)) + self._fields = [query.record().fieldName(i) for i in columns_to_filter] + + querystr = 'select * from {} where '.format(self.tablename) + querystr += ' or '.join('{} like ?'.format(f) for f in self._fields) + querystr += " escape '\\'" + + if sort_by: + sortstr = 'asc' if sort_order == Qt.AscendingOrder else 'desc' + querystr += ' order by {} {}'.format(sort_by, sortstr) + + if limit: + querystr += ' limit {}'.format(limit) + + self._querystr = querystr + self.set_pattern('%') + + def set_pattern(self, pattern): + # TODO: kill star-args for run_query + query = sql.run_query(self._querystr, *[pattern for _ in self._fields]) + self.setQuery(query) class SqlCompletionModel(QAbstractItemModel): @@ -58,7 +88,7 @@ class SqlCompletionModel(QAbstractItemModel): self.srcmodel = self # TODO: dummy for compat with old API self.pattern = '' - def new_category(self, name, sort_by=None, sort_order=Qt.AscendingOrder): + def new_category(self, name, sort_by=None, sort_order=None, limit=None): """Create a new completion category and add it to this model. Args: @@ -68,14 +98,10 @@ class SqlCompletionModel(QAbstractItemModel): Return: A new CompletionCategory. """ - database = QSqlDatabase.database() - cat = QSqlTableModel(parent=self, db=database) - cat.setTable(name) - if sort_by: - cat.setSort(cat.fieldIndex(sort_by), sort_order) - cat.select() + cat = SqlCompletionCategory(name, parent=self, sort_by=sort_by, + sort_order=sort_order, limit=limit, + columns_to_filter=self.columns_to_filter) self._categories.append(cat) - return cat def delete_cur_item(self, completion): """Delete the selected item.""" @@ -95,7 +121,7 @@ class SqlCompletionModel(QAbstractItemModel): return if not index.parent().isValid(): if index.column() == 0: - return self._categories[index.row()].tableName() + return self._categories[index.row()].tablename else: table = self._categories[index.parent().row()] idx = table.index(index.row(), index.column()) @@ -177,6 +203,7 @@ class SqlCompletionModel(QAbstractItemModel): Args: pattern: The filter pattern to set. """ + log.completion.debug("Setting completion pattern '{}'".format(pattern)) # TODO: should pattern be saved in the view layer instead? self.pattern = pattern # escape to treat a user input % or _ as a literal, not a wildcard @@ -184,14 +211,9 @@ class SqlCompletionModel(QAbstractItemModel): pattern = pattern.replace('_', '\\_') # treat spaces as wildcards to match any of the typed words pattern = re.sub(r' +', '%', pattern) - for t in self._categories: - fields = (t.record().fieldName(i) for i in self.columns_to_filter) - query = ' or '.join("{} like '%{}%' escape '\\'" - .format(field, pattern) - for field in fields) - log.completion.debug("Setting filter = '{}' for table '{}'" - .format(query, t.tableName())) - t.setFilter(query) + pattern = '%{}%'.format(pattern) + for cat in self._categories: + cat.set_pattern(pattern) def first_item(self): """Return the index of the first child (non-category) in the model.""" diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index fd7257bdb..9065caa48 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -39,7 +39,7 @@ def close(): QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) -def _run_query(querystr, *values): +def run_query(querystr, *values): """Run the given SQL query string on the database. Args: @@ -85,12 +85,12 @@ class SqlTable(QObject): super().__init__(parent) self._name = name self._primary_key = primary_key - _run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))" + run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))" .format(name, ','.join(fields), primary_key)) def __iter__(self): """Iterate rows in the table.""" - result = _run_query("SELECT * FROM {}".format(self._name)) + result = run_query("SELECT * FROM {}".format(self._name)) while result.next(): rec = result.record() yield tuple(rec.value(i) for i in range(rec.count())) @@ -101,13 +101,13 @@ class SqlTable(QObject): Args: key: Primary key value to search for. """ - query = _run_query("SELECT * FROM {} where {} = ?" + query = run_query("SELECT * FROM {} where {} = ?" .format(self._name, self._primary_key), key) return query.next() def __len__(self): """Return the count of rows in the table.""" - result = _run_query("SELECT count(*) FROM {}".format(self._name)) + result = run_query("SELECT count(*) FROM {}".format(self._name)) result.next() return result.value(0) @@ -117,7 +117,7 @@ class SqlTable(QObject): Args: key: Primary key value to fetch. """ - result = _run_query("SELECT * FROM {} where {} = ?" + result = run_query("SELECT * FROM {} where {} = ?" .format(self._name, self._primary_key), key) result.next() rec = result.record() @@ -134,7 +134,7 @@ class SqlTable(QObject): The number of rows deleted. """ field = field or self._primary_key - query = _run_query("DELETE FROM {} where {} = ?" + query = run_query("DELETE FROM {} where {} = ?" .format(self._name, field), value) if not query.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) @@ -149,13 +149,13 @@ class SqlTable(QObject): """ cmd = "REPLACE" if replace else "INSERT" paramstr = ','.join(['?'] * len(values)) - _run_query("{} INTO {} values({})" + run_query("{} INTO {} values({})" .format(cmd, self._name, paramstr), *values) self.changed.emit() def delete_all(self): """Remove all row from the table.""" - _run_query("DELETE FROM {}".format(self._name)) + run_query("DELETE FROM {}".format(self._name)) self.changed.emit() diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 49cb216dd..024b69ec2 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -155,16 +155,19 @@ def test_sorting(sort_by, sort_order, data, expected): ('_', [0], [('A', [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')])], [('A', [('a_b', '', ''), ('__a', '', '')])]), + + ("can't", [0], + [('A', [("can't touch this", '', ''), ('a', '', '')])], + [('A', [("can't touch this", '', '')])]), ]) def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" - model = sqlmodel.SqlCompletionModel() + model = sqlmodel.SqlCompletionModel(columns_to_filter=filter_cols) for name, rows in before: table = sql.SqlTable(name, ['a', 'b', 'c'], primary_key='a') for row in rows: table.insert(*row) model.new_category(name) - model.columns_to_filter = filter_cols model.set_pattern(pattern) _check_model(model, after) @@ -198,3 +201,11 @@ def test_first_last_item(data, first, last): model.new_category(name) assert model.data(model.first_item()) == first assert model.data(model.last_item()) == last + +def test_limit(): + table = sql.SqlTable('test_limit', ['a'], primary_key='a') + for i in range(5): + table.insert([i]) + model = sqlmodel.SqlCompletionModel() + model.new_category('test_limit', limit=3) + assert model.count() == 3 From df995c02a38b3542dc528d9679d9557188c81f01 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 08:59:01 -0500 Subject: [PATCH 019/161] Get rid of varargs in sql.run_query. Things are clearer when just passing a list. --- qutebrowser/browser/history.py | 4 +- qutebrowser/browser/urlmarks.py | 8 ++-- qutebrowser/completion/models/sqlmodel.py | 3 +- qutebrowser/misc/sql.py | 18 ++++----- tests/unit/completion/test_models.py | 24 ++++++------ tests/unit/completion/test_sqlmodel.py | 8 ++-- tests/unit/misc/test_sql.py | 46 +++++++++++------------ 7 files changed, 55 insertions(+), 56 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 955de21f2..d2adea771 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -188,8 +188,8 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" - self.insert(entry.url_str(), entry.title, entry.atime, entry.redirect, - replace=True) + self.insert([entry.url_str(), entry.title, entry.atime, + entry.redirect], replace=True) def get_recent(self): """Get the most recent history entries.""" diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index 6d692da0a..e28966b24 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -128,7 +128,7 @@ class QuickmarkManager(UrlMarkManager): except ValueError: message.error("Invalid quickmark '{}'".format(line)) else: - self.insert(key, url) + self.insert([key, url]) def prompt_save(self, url): """Prompt for a new quickmark name to be added and add it. @@ -168,7 +168,7 @@ class QuickmarkManager(UrlMarkManager): def set_mark(): """Really set the quickmark.""" - self.insert(name, url) + self.insert([name, url]) log.misc.debug("Added quickmark {} for {}".format(name, url)) if name in self: @@ -231,7 +231,7 @@ class BookmarkManager(UrlMarkManager): parts = line.split(maxsplit=1) urlstr = parts[0] title = parts[1] if len(parts) == 2 else '' - self.insert(urlstr, title) + self.insert([urlstr, title]) def add(self, url, title, *, toggle=False): """Add a new bookmark. @@ -259,5 +259,5 @@ class BookmarkManager(UrlMarkManager): else: raise AlreadyExistsError("Bookmark already exists!") else: - self.insert(urlstr, title) + self.insert([urlstr, title]) return True diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 40b2e27ca..b45990e6d 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -52,8 +52,7 @@ class SqlCompletionCategory(QSqlQueryModel): self.set_pattern('%') def set_pattern(self, pattern): - # TODO: kill star-args for run_query - query = sql.run_query(self._querystr, *[pattern for _ in self._fields]) + query = sql.run_query(self._querystr, [pattern for _ in self._fields]) self.setQuery(query) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 9065caa48..11d64bf72 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -39,17 +39,17 @@ def close(): QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) -def run_query(querystr, *values): +def run_query(querystr, values=None): """Run the given SQL query string on the database. Args: - values: positional parameter bindings. + values: A list of positional parameter bindings. """ log.completion.debug('Running SQL query: "{}"'.format(querystr)) database = QSqlDatabase.database() query = QSqlQuery(database) query.prepare(querystr) - for val in values: + for val in values or []: query.addBindValue(val) log.completion.debug('Query bindings: {}'.format(query.boundValues())) if not query.exec_(): @@ -102,7 +102,7 @@ class SqlTable(QObject): key: Primary key value to search for. """ query = run_query("SELECT * FROM {} where {} = ?" - .format(self._name, self._primary_key), key) + .format(self._name, self._primary_key), [key]) return query.next() def __len__(self): @@ -118,7 +118,7 @@ class SqlTable(QObject): key: Primary key value to fetch. """ result = run_query("SELECT * FROM {} where {} = ?" - .format(self._name, self._primary_key), key) + .format(self._name, self._primary_key), [key]) result.next() rec = result.record() return tuple(rec.value(i) for i in range(rec.count())) @@ -135,22 +135,22 @@ class SqlTable(QObject): """ field = field or self._primary_key query = run_query("DELETE FROM {} where {} = ?" - .format(self._name, field), value) + .format(self._name, field), [value]) if not query.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, *values, replace=False): + def insert(self, values, replace=False): """Append a row to the table. Args: - values: Values in the order fields were given on table creation. + values: A list of values to insert. replace: If true, allow inserting over an existing primary key. """ cmd = "REPLACE" if replace else "INSERT" paramstr = ','.join(['?'] * len(values)) run_query("{} INTO {} values({})" - .format(cmd, self._name, paramstr), *values) + .format(cmd, self._name, paramstr), values) self.changed.emit() def delete_all(self): diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index e1366325c..a89041713 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -135,18 +135,18 @@ def _mock_view_index(model, category_idx, child_idx, qtbot): def quickmarks(init_sql): """Pre-populate the quickmark database.""" table = sql.SqlTable('Quickmarks', ['name', 'url'], primary_key='name') - table.insert('aw', 'https://wiki.archlinux.org') - table.insert('ddg', 'https://duckduckgo.com') - table.insert('wiki', 'https://wikipedia.org') + table.insert(['aw', 'https://wiki.archlinux.org']) + table.insert(['ddg', 'https://duckduckgo.com']) + table.insert(['wiki', 'https://wikipedia.org']) @pytest.fixture def bookmarks(init_sql): """Pre-populate the bookmark database.""" table = sql.SqlTable('Bookmarks', ['url', 'title'], primary_key='url') - table.insert('https://github.com', 'GitHub') - table.insert('https://python.org', 'Welcome to Python.org') - table.insert('http://qutebrowser.org', 'qutebrowser | qutebrowser') + table.insert(['https://github.com', 'GitHub']) + table.insert(['https://python.org', 'Welcome to Python.org']) + table.insert(['http://qutebrowser.org', 'qutebrowser | qutebrowser']) @pytest.fixture @@ -154,12 +154,12 @@ def web_history(stubs, init_sql): """Pre-populate the web-history database.""" table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect'], primary_key='url') - table.insert('http://qutebrowser.org', 'qutebrowser', - datetime(2015, 9, 5).timestamp(), False) - table.insert('https://python.org', 'Welcome to Python.org', - datetime(2016, 3, 8).timestamp(), False) - table.insert('https://github.com', 'https://github.com', - datetime(2016, 5, 1).timestamp(), False) + table.insert(['http://qutebrowser.org', 'qutebrowser', + datetime(2015, 9, 5).timestamp(), False]) + table.insert(['https://python.org', 'Welcome to Python.org', + datetime(2016, 3, 8).timestamp(), False]) + table.insert(['https://github.com', 'https://github.com', + datetime(2016, 5, 1).timestamp(), False]) def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 024b69ec2..2c158d3d4 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -67,7 +67,7 @@ def test_count(rowcounts, expected): name = 'Foo' + str(i) table = sql.SqlTable(name, ['a'], primary_key='a') for rownum in range(rowcount): - table.insert(rownum) + table.insert([rownum]) model.new_category(name) assert model.count() == expected @@ -104,7 +104,7 @@ def test_count(rowcounts, expected): def test_sorting(sort_by, sort_order, data, expected): table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') for row in data: - table.insert(*row) + table.insert(row) model = sqlmodel.SqlCompletionModel() model.new_category('Foo', sort_by=sort_by, sort_order=sort_order) _check_model(model, [('Foo', expected)]) @@ -166,7 +166,7 @@ def test_set_pattern(pattern, filter_cols, before, after): for name, rows in before: table = sql.SqlTable(name, ['a', 'b', 'c'], primary_key='a') for row in rows: - table.insert(*row) + table.insert(row) model.new_category(name) model.set_pattern(pattern) _check_model(model, after) @@ -197,7 +197,7 @@ def test_first_last_item(data, first, last): for name, rows in data: table = sql.SqlTable(name, ['a'], primary_key='a') for row in rows: - table.insert(row) + table.insert([row]) model.new_category(name) assert model.data(model.first_item()) == first assert model.data(model.last_item()) == last diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 889e28a19..4a5299b48 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -36,19 +36,19 @@ def test_init(): def test_insert(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') with qtbot.waitSignal(table.changed): - table.insert('one', 1, False) + table.insert(['one', 1, False]) with qtbot.waitSignal(table.changed): - table.insert('wan', 1, False) + table.insert(['wan', 1, False]) with pytest.raises(sql.SqlException): # duplicate primary key - table.insert('one', 1, False) + table.insert(['one', 1, False]) def test_iter(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) - table.insert('nine', 9, False) - table.insert('thirteen', 13, True) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) assert list(table) == [('one', 1, False), ('nine', 9, False), ('thirteen', 13, True)] @@ -56,17 +56,17 @@ def test_iter(): def test_replace(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) + table.insert(['one', 1, False]) with qtbot.waitSignal(table.changed): - table.insert('one', 1, True, replace=True) + table.insert(['one', 1, True], replace=True) assert list(table) == [('one', 1, True)] def test_delete(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) - table.insert('nine', 9, False) - table.insert('thirteen', 13, True) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) with pytest.raises(KeyError): table.delete('nope') with qtbot.waitSignal(table.changed): @@ -80,19 +80,19 @@ def test_delete(qtbot): def test_len(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') assert len(table) == 0 - table.insert('one', 1, False) + table.insert(['one', 1, False]) assert len(table) == 1 - table.insert('nine', 9, False) + table.insert(['nine', 9, False]) assert len(table) == 2 - table.insert('thirteen', 13, True) + table.insert(['thirteen', 13, True]) assert len(table) == 3 def test_contains(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) - table.insert('nine', 9, False) - table.insert('thirteen', 13, True) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) assert 'oone' not in table assert 'ninee' not in table assert 1 not in table @@ -104,9 +104,9 @@ def test_contains(): def test_index(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) - table.insert('nine', 9, False) - table.insert('thirteen', 13, True) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) assert table['one'] == ('one', 1, False) assert table['nine'] == ('nine', 9, False) assert table['thirteen'] == ('thirteen', 13, True) @@ -114,9 +114,9 @@ def test_index(): def test_delete_all(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) - table.insert('nine', 9, False) - table.insert('thirteen', 13, True) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) with qtbot.waitSignal(table.changed): table.delete_all() assert list(table) == [] From 3f6f03e3250194672f8aa87c8973238d6d382797 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 09:06:42 -0500 Subject: [PATCH 020/161] Respect web-history-max-items with SQL. Use a LIMIT with the sql competion model to limit history entries as the old completion implementation did. --- qutebrowser/completion/models/urlmodel.py | 4 +++- tests/unit/completion/test_models.py | 6 ++---- tests/unit/completion/test_sqlmodel.py | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 1390fff21..b844fa93e 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -20,6 +20,7 @@ """Function to return the url completion model for the `open` command.""" from qutebrowser.completion.models import sqlmodel +from qutebrowser.config import config def url(): @@ -32,7 +33,8 @@ def url(): model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), columns_to_filter=[urlcol, textcol]) - model.new_category('History') + model.new_category('History', + limit=config.get('completion', 'web-history-max-items')) model.new_category('Quickmarks') model.new_category('Bookmarks') return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index a89041713..91ac8286f 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -268,8 +268,8 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, - the most recent entries are included """ # TODO: time formatting and item limiting - #config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - # 'web-history-max-items': 2} + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', + 'web-history-max-items': 2} model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -303,8 +303,6 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, datetime(2015, 9, 5).timestamp()), ('https://python.org', 'Welcome to Python.org', datetime(2016, 3, 8).timestamp()), - ('https://github.com', 'https://github.com', - datetime(2016, 5, 1).timestamp()), ], }) diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 2c158d3d4..1044a831f 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -202,6 +202,7 @@ def test_first_last_item(data, first, last): assert model.data(model.first_item()) == first assert model.data(model.last_item()) == last + def test_limit(): table = sql.SqlTable('test_limit', ['a'], primary_key='a') for i in range(5): From 9f27a9a5d77a5bdd63d9c6a1d1f52e8429d87a97 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 20:32:44 -0500 Subject: [PATCH 021/161] Implement column selectors for sql completion. A SQL completion category can now provide a customized column expression for the select statement. This enables the url model to format timestamps, as well as rearrange the name and url in the quickmark section. --- qutebrowser/completion/models/sqlmodel.py | 12 +++++++---- qutebrowser/completion/models/urlmodel.py | 8 ++++++-- tests/unit/completion/test_models.py | 25 +++++------------------ tests/unit/completion/test_sqlmodel.py | 8 ++++++++ 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index b45990e6d..8a0abda2b 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -29,15 +29,15 @@ from qutebrowser.misc import sql class SqlCompletionCategory(QSqlQueryModel): - def __init__(self, name, sort_by, sort_order, limit, columns_to_filter, - parent=None): + def __init__(self, name, sort_by, sort_order, limit, select, + columns_to_filter, parent=None): super().__init__(parent=parent) self.tablename = name query = sql.run_query('select * from {} limit 1'.format(name)) self._fields = [query.record().fieldName(i) for i in columns_to_filter] - querystr = 'select * from {} where '.format(self.tablename) + querystr = 'select {} from {} where '.format(select, name) querystr += ' or '.join('{} like ?'.format(f) for f in self._fields) querystr += " escape '\\'" @@ -87,18 +87,22 @@ class SqlCompletionModel(QAbstractItemModel): self.srcmodel = self # TODO: dummy for compat with old API self.pattern = '' - def new_category(self, name, sort_by=None, sort_order=None, limit=None): + def new_category(self, name, select='*', sort_by=None, + sort_order=None, limit=None): """Create a new completion category and add it to this model. Args: name: Name of category, and the table in the database. + select: A custom result column expression for the select statement. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Sorting order, if sort_by is non-None. + limit: Maximum row count to return on a query. Return: A new CompletionCategory. """ cat = SqlCompletionCategory(name, parent=self, sort_by=sort_by, sort_order=sort_order, limit=limit, + select=select, columns_to_filter=self.columns_to_filter) self._categories.append(cat) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index b844fa93e..164a5da15 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -33,8 +33,12 @@ def url(): model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), columns_to_filter=[urlcol, textcol]) + limit = config.get('completion', 'web-history-max-items') + timefmt = config.get('completion', 'timestamp-format') + select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) model.new_category('History', - limit=config.get('completion', 'web-history-max-items')) - model.new_category('Quickmarks') + limit=limit, + select='url, title, {}'.format(select_time)) + model.new_category('Quickmarks', select='url, name') model.new_category('Bookmarks') return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 91ac8286f..21b65e092 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -275,34 +275,19 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, qtmodeltester.check(model) _check_completions(model, { - # TODO: rearrange columns so address comes first - #"Quickmarks": [ - # ('https://wiki.archlinux.org', 'aw', None), - # ('https://duckduckgo.com', 'ddg', None), - # ('https://wikipedia.org', 'wiki', None), - #], "Quickmarks": [ - ('aw', 'https://wiki.archlinux.org', None), - ('ddg', 'https://duckduckgo.com', None), - ('wiki', 'https://wikipedia.org', None), + ('https://wiki.archlinux.org', 'aw', None), + ('https://duckduckgo.com', 'ddg', None), + ('https://wikipedia.org', 'wiki', None), ], "Bookmarks": [ ('https://github.com', 'GitHub', None), ('https://python.org', 'Welcome to Python.org', None), ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ], - # TODO: time formatting and item limiting - #"History": [ - # ('https://python.org', 'Welcome to Python.org', '2016-03-08'), - # ('https://github.com', 'GitHub', '2016-05-01'), - # ('http://qutebrowser.org', 'qutebrowser | qutebrowser', - # '2015-09-05', False) - #], "History": [ - ('http://qutebrowser.org', 'qutebrowser', - datetime(2015, 9, 5).timestamp()), - ('https://python.org', 'Welcome to Python.org', - datetime(2016, 3, 8).timestamp()), + ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ('https://python.org', 'Welcome to Python.org', '2016-03-08'), ], }) diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 1044a831f..a0dbbe7b3 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -210,3 +210,11 @@ def test_limit(): model = sqlmodel.SqlCompletionModel() model.new_category('test_limit', limit=3) assert model.count() == 3 + + +def test_select(): + table = sql.SqlTable('test_select', ['a', 'b', 'c'], primary_key='a') + table.insert(['foo', 'bar', 'baz']) + model = sqlmodel.SqlCompletionModel() + model.new_category('test_select', select='b, c, a') + _check_model(model, [('test_select', [('bar', 'baz', 'foo')])]) From fe808787887638ead2dba61c7a7ff16760895833 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 22:10:49 -0500 Subject: [PATCH 022/161] Implement custom where clause in SQL. Allow categories to specify a WHERE clause that applies in addition to the pattern filter. This allows the url completion model to filter out redirect entries. This also fixed the usage of ESCAPE so it applies to all the LIKE statements. --- qutebrowser/completion/models/sqlmodel.py | 16 ++++++++++------ qutebrowser/completion/models/urlmodel.py | 3 ++- tests/unit/completion/test_models.py | 3 +++ tests/unit/completion/test_sqlmodel.py | 13 +++++++++++++ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 8a0abda2b..39182628c 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -29,7 +29,7 @@ from qutebrowser.misc import sql class SqlCompletionCategory(QSqlQueryModel): - def __init__(self, name, sort_by, sort_order, limit, select, + def __init__(self, name, sort_by, sort_order, limit, select, where, columns_to_filter, parent=None): super().__init__(parent=parent) self.tablename = name @@ -37,9 +37,12 @@ class SqlCompletionCategory(QSqlQueryModel): query = sql.run_query('select * from {} limit 1'.format(name)) self._fields = [query.record().fieldName(i) for i in columns_to_filter] - querystr = 'select {} from {} where '.format(select, name) - querystr += ' or '.join('{} like ?'.format(f) for f in self._fields) - querystr += " escape '\\'" + querystr = 'select {} from {} where ('.format(select, name) + querystr += ' or '.join("{} like ? escape '\\'".format(f) + for f in self._fields) + querystr += ')' + if where: + querystr += ' and ' + where if sort_by: sortstr = 'asc' if sort_order == Qt.AscendingOrder else 'desc' @@ -87,13 +90,14 @@ class SqlCompletionModel(QAbstractItemModel): self.srcmodel = self # TODO: dummy for compat with old API self.pattern = '' - def new_category(self, name, select='*', sort_by=None, + def new_category(self, name, select='*', where=None, sort_by=None, sort_order=None, limit=None): """Create a new completion category and add it to this model. Args: name: Name of category, and the table in the database. select: A custom result column expression for the select statement. + where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Sorting order, if sort_by is non-None. limit: Maximum row count to return on a query. @@ -102,7 +106,7 @@ class SqlCompletionModel(QAbstractItemModel): """ cat = SqlCompletionCategory(name, parent=self, sort_by=sort_by, sort_order=sort_order, limit=limit, - select=select, + select=select, where=where, columns_to_filter=self.columns_to_filter) self._categories.append(cat) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 164a5da15..2f573aab7 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -38,7 +38,8 @@ def url(): select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) model.new_category('History', limit=limit, - select='url, title, {}'.format(select_time)) + select='url, title, {}'.format(select_time), + where='not redirect') model.new_category('Quickmarks', select='url, name') model.new_category('Bookmarks') return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 21b65e092..d0732f660 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -154,6 +154,8 @@ def web_history(stubs, init_sql): """Pre-populate the web-history database.""" table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect'], primary_key='url') + table.insert(['http://some-redirect.example.com', 'redirect', + datetime(2016, 9, 5).timestamp(), True]) table.insert(['http://qutebrowser.org', 'qutebrowser', datetime(2015, 9, 5).timestamp(), False]) table.insert(['https://python.org', 'Welcome to Python.org', @@ -266,6 +268,7 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, - quickmarks, bookmarks, and urls are included - no more than 'web-history-max-items' items are included (TODO) - the most recent entries are included + - redirect entries are not included """ # TODO: time formatting and item limiting config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index a0dbbe7b3..b94e8e048 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -156,6 +156,10 @@ def test_sorting(sort_by, sort_order, data, expected): [('A', [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')])], [('A', [('a_b', '', ''), ('__a', '', '')])]), + ('%', [0, 1], + [('A', [('\\foo', '\\bar', '')])], + [('A', [])]), + ("can't", [0], [('A', [("can't touch this", '', ''), ('a', '', '')])], [('A', [("can't touch this", '', '')])]), @@ -218,3 +222,12 @@ def test_select(): model = sqlmodel.SqlCompletionModel() model.new_category('test_select', select='b, c, a') _check_model(model, [('test_select', [('bar', 'baz', 'foo')])]) + + +def test_where(): + table = sql.SqlTable('test_where', ['a', 'b', 'c'], primary_key='a') + table.insert(['foo', 'bar', False]) + table.insert(['baz', 'biz', True]) + model = sqlmodel.SqlCompletionModel() + model.new_category('test_where', where='not c') + _check_model(model, [('test_where', [('foo', 'bar', False)])]) From d89898ef7d83c7bf26d1215e58d172c47e08b39c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 22:43:33 -0500 Subject: [PATCH 023/161] Implement delete_cur_item for sql completion. This re-enables bookmark/quickmark deletion for url completions via the new SQL interface. --- qutebrowser/completion/models/sqlmodel.py | 7 ++-- qutebrowser/completion/models/urlmodel.py | 43 +++++++++++++++++++---- tests/unit/completion/test_models.py | 28 +++++++++------ 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 39182628c..dc66f3480 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -82,13 +82,14 @@ class SqlCompletionModel(QAbstractItemModel): """ def __init__(self, column_widths=(30, 70, 0), columns_to_filter=None, - parent=None): + delete_cur_item=None, parent=None): super().__init__(parent) self.columns_to_filter = columns_to_filter or [0] self.column_widths = column_widths self._categories = [] self.srcmodel = self # TODO: dummy for compat with old API self.pattern = '' + self.delete_cur_item = delete_cur_item def new_category(self, name, select='*', where=None, sort_by=None, sort_order=None, limit=None): @@ -110,10 +111,6 @@ class SqlCompletionModel(QAbstractItemModel): columns_to_filter=self.columns_to_filter) self._categories.append(cat) - def delete_cur_item(self, completion): - """Delete the selected item.""" - raise NotImplementedError - def data(self, index, role=Qt.DisplayRole): """Return the item data for index. diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 2f573aab7..161a9266b 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -21,6 +21,39 @@ from qutebrowser.completion.models import sqlmodel from qutebrowser.config import config +from qutebrowser.utils import qtutils, log, objreg + + +_URLCOL = 0 +_TEXTCOL = 1 + + +def _delete_url(completion): + """Delete the selected item. + + Args: + completion: The Completion object to use. + """ + index = completion.currentIndex() + qtutils.ensure_valid(index) + category = index.parent() + index = category.child(index.row(), _URLCOL) + catname = category.data() + url = index.data() + qtutils.ensure_valid(category) + + if catname == 'Bookmarks': + log.completion.debug('Deleting bookmark {}'.format(url)) + bookmark_manager = objreg.get('bookmark-manager') + bookmark_manager.delete(url) + else: + assert catname == 'Quickmarks', 'Unknown category {}'.format(catname) + quickmark_manager = objreg.get('quickmark-manager') + sibling = index.sibling(index.row(), _TEXTCOL) + qtutils.ensure_valid(sibling) + name = sibling.data() + log.completion.debug('Deleting quickmark {}'.format(name)) + quickmark_manager.delete(name) def url(): @@ -28,18 +61,16 @@ def url(): Used for the `open` command. """ - urlcol = 0 - textcol = 1 - model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), - columns_to_filter=[urlcol, textcol]) + columns_to_filter=[_URLCOL, _TEXTCOL], + delete_cur_item=_delete_url) limit = config.get('completion', 'web-history-max-items') timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) + model.new_category('Quickmarks', select='url, name') + model.new_category('Bookmarks') model.new_category('History', limit=limit, select='url, title, {}'.format(select_time), where='not redirect') - model.new_category('Quickmarks', select='url, name') - model.new_category('Bookmarks') return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index d0732f660..ddc746aa6 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -30,6 +30,7 @@ from qutebrowser.browser import history from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value from qutebrowser.misc import sql +from qutebrowser.utils import objreg def _check_completions(model, expected): @@ -115,7 +116,7 @@ def _patch_config_section_desc(monkeypatch, stubs, symbol): monkeypatch.setattr(symbol, section_desc) -def _mock_view_index(model, category_idx, child_idx, qtbot): +def _mock_view_index(model, category_num, child_num, qtbot): """Create a tree view from a model and set the current index. Args: @@ -126,8 +127,9 @@ def _mock_view_index(model, category_idx, child_idx, qtbot): view = QTreeView() qtbot.add_widget(view) view.setModel(model) - idx = model.indexFromItem(model.item(category_idx).child(child_idx)) - view.setCurrentIndex(idx) + parent = model.index(category_num, 0) + child = model.index(child_num, 0, parent=parent) + view.setCurrentIndex(child) return view @@ -138,6 +140,9 @@ def quickmarks(init_sql): table.insert(['aw', 'https://wiki.archlinux.org']) table.insert(['ddg', 'https://duckduckgo.com']) table.insert(['wiki', 'https://wikipedia.org']) + objreg.register('quickmark-manager', table) + yield table + objreg.delete('quickmark-manager') @pytest.fixture @@ -147,6 +152,9 @@ def bookmarks(init_sql): table.insert(['https://github.com', 'GitHub']) table.insert(['https://python.org', 'Welcome to Python.org']) table.insert(['http://qutebrowser.org', 'qutebrowser | qutebrowser']) + objreg.register('bookmark-manager', table) + yield table + objreg.delete('bookmark-manager') @pytest.fixture @@ -295,7 +303,6 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, }) -@pytest.mark.skip def test_url_completion_delete_bookmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): @@ -309,12 +316,11 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, # delete item (1, 0) -> (bookmarks, 'https://github.com' ) view = _mock_view_index(model, 1, 0, qtbot) model.delete_cur_item(view) - assert 'https://github.com' not in bookmarks.marks - assert 'https://python.org' in bookmarks.marks - assert 'http://qutebrowser.org' in bookmarks.marks + assert 'https://github.com' not in bookmarks + assert 'https://python.org' in bookmarks + assert 'http://qutebrowser.org' in bookmarks -@pytest.mark.skip def test_url_completion_delete_quickmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): @@ -328,9 +334,9 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, # delete item (0, 1) -> (quickmarks, 'ddg' ) view = _mock_view_index(model, 0, 1, qtbot) model.delete_cur_item(view) - assert 'aw' in quickmarks.marks - assert 'ddg' not in quickmarks.marks - assert 'wiki' in quickmarks.marks + assert 'aw' in quickmarks + assert 'ddg' not in quickmarks + assert 'wiki' in quickmarks def test_session_completion(qtmodeltester, session_manager_stub): From 0e650ad719ecc9b635dc53865073c65732b4bceb Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 09:06:23 -0500 Subject: [PATCH 024/161] Return namedtuples from SqlTable. Instead of returning a regular tuple and trying to remember which index maps to which field, return named tuples that allow accessing the fields by name. --- qutebrowser/browser/urlmarks.py | 2 +- qutebrowser/misc/sql.py | 7 +++++-- tests/unit/completion/test_sqlmodel.py | 6 ++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index e28966b24..d9e49b5c2 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -193,7 +193,7 @@ class QuickmarkManager(UrlMarkManager): if name not in self: raise DoesNotExistError("Quickmark '{}' does not exist!" .format(name)) - urlstr = self[name] + urlstr = self[name].url try: url = urlutils.fuzzy_url(urlstr, do_search=False) except urlutils.InvalidUrlError as e: diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 11d64bf72..f23d791ad 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -24,6 +24,8 @@ from PyQt5.QtSql import QSqlDatabase, QSqlQuery from qutebrowser.utils import log +import collections + def init(): """Initialize the SQL database connection.""" @@ -87,13 +89,14 @@ class SqlTable(QObject): self._primary_key = primary_key run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))" .format(name, ','.join(fields), primary_key)) + self.Entry = collections.namedtuple(name + '_Entry', fields) def __iter__(self): """Iterate rows in the table.""" result = run_query("SELECT * FROM {}".format(self._name)) while result.next(): rec = result.record() - yield tuple(rec.value(i) for i in range(rec.count())) + yield self.Entry(*[rec.value(i) for i in range(rec.count())]) def __contains__(self, key): """Return whether the table contains the matching item. @@ -121,7 +124,7 @@ class SqlTable(QObject): .format(self._name, self._primary_key), [key]) result.next() rec = result.record() - return tuple(rec.value(i) for i in range(rec.count())) + return self.Entry(*[rec.value(i) for i in range(rec.count())]) def delete(self, value, field=None): """Remove all rows for which `field` equals `value`. diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index b94e8e048..d3e3ada96 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -231,3 +231,9 @@ def test_where(): model = sqlmodel.SqlCompletionModel() model.new_category('test_where', where='not c') _check_model(model, [('test_where', [('foo', 'bar', False)])]) + +def test_entry(): + table = sql.SqlTable('test_entry', ['a', 'b', 'c'], primary_key='a') + assert hasattr(table.Entry, 'a') + assert hasattr(table.Entry, 'b') + assert hasattr(table.Entry, 'c') From ea9217a61f9b49940dcb2d82fd595b0cb6e16f32 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 09:07:07 -0500 Subject: [PATCH 025/161] Fix qutescheme for new SQL backend. The qute://history and qute://bookmarks handlers were added during my work, and had to be adapted to the SQL-based history backend. --- qutebrowser/browser/qutescheme.py | 2 +- tests/unit/browser/test_qutescheme.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index b0c555e4e..23b009f6d 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -198,7 +198,7 @@ def history_data(start_time): # noqa Arguments: reverse -- whether to reverse the history_dict before iterating. """ - history = objreg.get('web-history').history_dict.values() + history = list(objreg.get('web-history')) if reverse: history = reversed(history) diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 92ad30574..d5e60efab 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -96,7 +96,7 @@ class TestHistoryHandler: return items @pytest.fixture - def fake_web_history(self, fake_save_manager, tmpdir): + def fake_web_history(self, fake_save_manager, tmpdir, init_sql): """Create a fake web-history and register it into objreg.""" web_history = history.WebHistory(tmpdir.dirname, 'fake-history') objreg.register('web-history', web_history) From ffd044b52b1ad07f7f9c62d70e505e5367da9b72 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 12:46:37 -0500 Subject: [PATCH 026/161] Fix pylint and flake8 for SQL work. Vanity commit. This also touches up a few comments. --- qutebrowser/browser/history.py | 2 +- qutebrowser/completion/models/sqlmodel.py | 12 ++++++------ qutebrowser/completion/models/urlmodel.py | 6 +++--- qutebrowser/misc/sql.py | 22 ++++++++++++---------- tests/unit/completion/test_completer.py | 2 +- tests/unit/completion/test_sqlmodel.py | 1 + 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index d2adea771..b6d328942 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -36,7 +36,7 @@ class Entry: Attributes: atime: The time the page was accessed. url: The URL which was accessed as QUrl. - redirect: If True, don't save this entry to disk + redirect: If True, don't show this entry in completion """ def __init__(self, atime, url, title, redirect=False): diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index dc66f3480..624cac80d 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -22,13 +22,13 @@ import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel -from PyQt5.QtSql import QSqlQuery, QSqlQueryModel, QSqlDatabase +from PyQt5.QtSql import QSqlQueryModel from qutebrowser.utils import log from qutebrowser.misc import sql -class SqlCompletionCategory(QSqlQueryModel): +class _SqlCompletionCategory(QSqlQueryModel): def __init__(self, name, sort_by, sort_order, limit, select, where, columns_to_filter, parent=None): super().__init__(parent=parent) @@ -105,10 +105,10 @@ class SqlCompletionModel(QAbstractItemModel): Return: A new CompletionCategory. """ - cat = SqlCompletionCategory(name, parent=self, sort_by=sort_by, - sort_order=sort_order, limit=limit, - select=select, where=where, - columns_to_filter=self.columns_to_filter) + cat = _SqlCompletionCategory(name, parent=self, sort_by=sort_by, + sort_order=sort_order, limit=limit, + select=select, where=where, + columns_to_filter=self.columns_to_filter) self._categories.append(cat) def data(self, index, role=Qt.DisplayRole): diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 161a9266b..387603dd5 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -39,13 +39,13 @@ def _delete_url(completion): category = index.parent() index = category.child(index.row(), _URLCOL) catname = category.data() - url = index.data() qtutils.ensure_valid(category) if catname == 'Bookmarks': - log.completion.debug('Deleting bookmark {}'.format(url)) + urlstr = index.data() + log.completion.debug('Deleting bookmark {}'.format(urlstr)) bookmark_manager = objreg.get('bookmark-manager') - bookmark_manager.delete(url) + bookmark_manager.delete(urlstr) else: assert catname == 'Quickmarks', 'Unknown category {}'.format(catname) quickmark_manager = objreg.get('quickmark-manager') diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index f23d791ad..53b23829f 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -65,6 +65,7 @@ class SqlTable(QObject): """Interface to a sql table. Attributes: + Entry: The class wrapping row data from this table. _name: Name of the SQL table this wraps. _primary_key: The primary key of the table. @@ -87,8 +88,9 @@ class SqlTable(QObject): super().__init__(parent) self._name = name self._primary_key = primary_key - run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))" - .format(name, ','.join(fields), primary_key)) + run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))".format( + name, ','.join(fields), primary_key)) + # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) def __iter__(self): @@ -104,8 +106,8 @@ class SqlTable(QObject): Args: key: Primary key value to search for. """ - query = run_query("SELECT * FROM {} where {} = ?" - .format(self._name, self._primary_key), [key]) + query = run_query("SELECT * FROM {} where {} = ?".format( + self._name, self._primary_key), [key]) return query.next() def __len__(self): @@ -120,8 +122,8 @@ class SqlTable(QObject): Args: key: Primary key value to fetch. """ - result = run_query("SELECT * FROM {} where {} = ?" - .format(self._name, self._primary_key), [key]) + result = run_query("SELECT * FROM {} where {} = ?".format( + self._name, self._primary_key), [key]) result.next() rec = result.record() return self.Entry(*[rec.value(i) for i in range(rec.count())]) @@ -137,8 +139,8 @@ class SqlTable(QObject): The number of rows deleted. """ field = field or self._primary_key - query = run_query("DELETE FROM {} where {} = ?" - .format(self._name, field), [value]) + query = run_query("DELETE FROM {} where {} = ?".format( + self._name, field), [value]) if not query.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() @@ -152,8 +154,8 @@ class SqlTable(QObject): """ cmd = "REPLACE" if replace else "INSERT" paramstr = ','.join(['?'] * len(values)) - run_query("{} INTO {} values({})" - .format(cmd, self._name, paramstr), values) + run_query("{} INTO {} values({})".format(cmd, self._name, paramstr), + values) self.changed.emit() def delete_all(self): diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index 34c9a2d19..dce105c27 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -36,7 +36,7 @@ class FakeCompletionModel(QStandardItemModel): def __init__(self, kind, *pos_args, parent=None): super().__init__(parent) self.kind = kind - self.pos_args = [*pos_args] + self.pos_args = list(pos_args) class CompletionWidgetStub(QObject): diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index d3e3ada96..f6daa722c 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -232,6 +232,7 @@ def test_where(): model.new_category('test_where', where='not c') _check_model(model, [('test_where', [('foo', 'bar', False)])]) + def test_entry(): table = sql.SqlTable('test_entry', ['a', 'b', 'c'], primary_key='a') assert hasattr(table.Entry, 'a') From 02fb1a037ce59027f500ccec2d74e0e843890999 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 20:55:27 -0500 Subject: [PATCH 027/161] Implement canFetchMore for SQL completion. This just forwards canFetchMore and fetchMore to the underlying tables. It seems to be returning True and fetching in some cases (with a large history), so I guess it is useful? --- qutebrowser/completion/models/sqlmodel.py | 31 +++++++++++++++++++++-- qutebrowser/misc/sql.py | 4 +-- qutebrowser/utils/log.py | 1 + 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 624cac80d..fcd8d37c6 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -91,6 +91,19 @@ class SqlCompletionModel(QAbstractItemModel): self.pattern = '' self.delete_cur_item = delete_cur_item + def _cat_from_idx(self, index): + """Return the category pointed to by the given index. + + Args: + idx: A QModelIndex + Returns: + A _SqlCompletionCategory if the index points at one, else None + """ + # items hold an index to the parent category in their internalPointer + # categories have an empty internalPointer + if index.isValid() and not index.internalPointer(): + return self._categories[index.row()] + def new_category(self, name, select='*', where=None, sort_by=None, sort_order=None, limit=None): """Create a new completion category and add it to this model. @@ -184,17 +197,31 @@ class SqlCompletionModel(QAbstractItemModel): if not parent.isValid(): # top-level return len(self._categories) - elif parent.internalPointer() or parent.column() != 0: + cat = self._cat_from_idx(parent) + if not cat or parent.column() != 0: # item or nonzero category column (only first col has children) return 0 else: # category - return self._categories[parent.row()].rowCount() + return cat.rowCount() def columnCount(self, parent=QModelIndex()): # pylint: disable=unused-argument return 3 + def canFetchMore(self, parent): + """Override to forward the call to the tables.""" + cat = self._cat_from_idx(parent) + if cat: + return cat.canFetchMore() + return False + + def fetchMore(self, parent): + """Override to forward the call to the tables.""" + cat = self._cat_from_idx(parent) + if cat: + cat.fetchMore() + def count(self): """Return the count of non-category items.""" return sum(t.rowCount() for t in self._categories) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 53b23829f..c051acf03 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -47,13 +47,13 @@ def run_query(querystr, values=None): Args: values: A list of positional parameter bindings. """ - log.completion.debug('Running SQL query: "{}"'.format(querystr)) + log.sql.debug('Running SQL query: "{}"'.format(querystr)) database = QSqlDatabase.database() query = QSqlQuery(database) query.prepare(querystr) for val in values or []: query.addBindValue(val) - log.completion.debug('Query bindings: {}'.format(query.boundValues())) + log.sql.debug('Query bindings: {}'.format(query.boundValues())) if not query.exec_(): raise SqlException('Failed to exec query "{}": "{}"'.format( querystr, query.lastError().text())) diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py index c2abbfb87..513c05062 100644 --- a/qutebrowser/utils/log.py +++ b/qutebrowser/utils/log.py @@ -141,6 +141,7 @@ sessions = logging.getLogger('sessions') webelem = logging.getLogger('webelem') prompt = logging.getLogger('prompt') network = logging.getLogger('network') +sql = logging.getLogger('sql') ram_handler = None From 6a04c4b3e898b258cd801b906bdd860058bf0f83 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 21:28:03 -0500 Subject: [PATCH 028/161] Allow replacing quickmark with SQL backend. This functionality was lost with the transition to SQL. The user should be able to replace a quickmark if they answer 'yes' to the prompt. --- qutebrowser/browser/urlmarks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index d9e49b5c2..5f7a4fb77 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -168,7 +168,7 @@ class QuickmarkManager(UrlMarkManager): def set_mark(): """Really set the quickmark.""" - self.insert([name, url]) + self.insert([name, url], replace=True) log.misc.debug("Added quickmark {} for {}".format(name, url)) if name in self: From be07107b1cd0d0eff91994d48c2932be811e584c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 21:29:00 -0500 Subject: [PATCH 029/161] Fix end2end completion tests for SQL backend. Change the logging to report the completion function name and have the end2end tests check for this. Remove the tests for realtime completion, as it was decided this is not an important feature and the code is much simpler without it. --- qutebrowser/completion/completer.py | 9 ++--- tests/end2end/features/completion.feature | 40 ++++--------------- tests/end2end/features/test_completion_bdd.py | 2 +- 3 files changed, 12 insertions(+), 39 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 8742156a3..f403354c3 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -97,6 +97,7 @@ class Completer(QObject): if not before_cursor: # '|' or 'set|' model = miscmodels.command() + log.completion.debug('Starting command completion') return sortfilter.CompletionFilterModel(source=model, parent=self) try: cmd = cmdutils.cmd_dict[before_cursor[0]] @@ -113,6 +114,8 @@ class Completer(QObject): if completion is None: return None model = self._get_completion_model(completion, before_cursor[1:]) + log.completion.debug('Starting {} completion' + .format(completion.__name__)) return model def _quote(self, s): @@ -237,11 +240,7 @@ class Completer(QObject): pattern = pattern.strip("'\"") model = self._get_new_completion(before_cursor, pattern) - - log.completion.debug("Setting completion model to {} with pattern '{}'" - .format(model.srcmodel.__class__.__name__ if model else 'None', - pattern)) - + log.completion.debug("Setting pattern to '{}'".format(pattern)) completion.set_model(model, pattern) def _change_completed_part(self, newtext, before, after, immediate=False): diff --git a/tests/end2end/features/completion.feature b/tests/end2end/features/completion.feature index b6c62336c..e93518199 100644 --- a/tests/end2end/features/completion.feature +++ b/tests/end2end/features/completion.feature @@ -32,23 +32,23 @@ Feature: Using completion Scenario: Using command completion When I run :set-cmd-text : - Then the completion model should be CommandCompletionModel + Then the completion model should be command Scenario: Using help completion When I run :set-cmd-text -s :help - Then the completion model should be HelpCompletionModel + Then the completion model should be helptopic Scenario: Using quickmark completion When I run :set-cmd-text -s :quickmark-load - Then the completion model should be QuickmarkCompletionModel + Then the completion model should be quickmark Scenario: Using bookmark completion When I run :set-cmd-text -s :bookmark-load - Then the completion model should be BookmarkCompletionModel + Then the completion model should be bookmark Scenario: Using bind completion When I run :set-cmd-text -s :bind X - Then the completion model should be BindCompletionModel + Then the completion model should be bind Scenario: Using session completion Given I open data/hello.txt @@ -62,37 +62,11 @@ Feature: Using completion Scenario: Using option completion When I run :set-cmd-text -s :set colors - Then the completion model should be SettingOptionCompletionModel + Then the completion model should be option Scenario: Using value completion When I run :set-cmd-text -s :set colors statusbar.bg - Then the completion model should be SettingValueCompletionModel - - Scenario: Updating the completion in realtime - Given I have a fresh instance - And I set completion -> quick-complete to false - When I open data/hello.txt - And I run :set-cmd-text -s :buffer - And I run :completion-item-focus next - And I open data/hello2.txt in a new background tab - And I run :completion-item-focus next - And I open data/hello3.txt in a new background tab - And I run :completion-item-focus next - And I run :command-accept - Then the following tabs should be open: - - data/hello.txt - - data/hello2.txt - - data/hello3.txt (active) - - Scenario: Updating the value completion in realtime - Given I set colors -> statusbar.bg to green - When I run :set-cmd-text -s :set colors statusbar.bg - And I set colors -> statusbar.bg to yellow - And I run :completion-item-focus next - And I run :completion-item-focus next - And I set colors -> statusbar.bg to red - And I run :command-accept - Then colors -> statusbar.bg should be yellow + Then the completion model should be value Scenario: Deleting an open tab via the completion Given I have a fresh instance diff --git a/tests/end2end/features/test_completion_bdd.py b/tests/end2end/features/test_completion_bdd.py index f4ada848f..4a06bfbc0 100644 --- a/tests/end2end/features/test_completion_bdd.py +++ b/tests/end2end/features/test_completion_bdd.py @@ -24,5 +24,5 @@ bdd.scenarios('completion.feature') @bdd.then(bdd.parsers.parse("the completion model should be {model}")) def check_model(quteproc, model): """Make sure the completion model was set to something.""" - pattern = "Setting completion model to {} with pattern *".format(model) + pattern = "Starting {} completion".format(model) quteproc.wait_for(message=pattern) From 490250f5bef8cc7b2bb9cb66601fa398a7fb2d3b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 07:39:50 -0500 Subject: [PATCH 030/161] Initialize SQL for two failing tests. test_selectors and test_get_all_objects were running fine on my machine, but for some reason is failing with "Driver not loaded" on Travis. Let's try initializing SQL and see what happens. --- tests/unit/browser/webkit/test_webkitelem.py | 2 +- tests/unit/utils/test_debug.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/browser/webkit/test_webkitelem.py b/tests/unit/browser/webkit/test_webkitelem.py index deff844fe..f6cd44a8c 100644 --- a/tests/unit/browser/webkit/test_webkitelem.py +++ b/tests/unit/browser/webkit/test_webkitelem.py @@ -211,7 +211,7 @@ class TestSelectorsAndFilters: assert self.TESTS @pytest.mark.parametrize('group, val, matching', TESTS) - def test_selectors(self, webframe, group, val, matching): + def test_selectors(self, webframe, init_sql, group, val, matching): webframe.setHtml('{}'.format(val)) # Make sure setting HTML succeeded and there's a new element assert len(webframe.findAllElements('*')) == 3 diff --git a/tests/unit/utils/test_debug.py b/tests/unit/utils/test_debug.py index 9b77b9628..646858163 100644 --- a/tests/unit/utils/test_debug.py +++ b/tests/unit/utils/test_debug.py @@ -244,7 +244,7 @@ class TestGetAllObjects: def __repr__(self): return '<{}>'.format(self._name) - def test_get_all_objects(self, stubs, monkeypatch): + def test_get_all_objects(self, stubs, monkeypatch, init_sql): # pylint: disable=unused-variable widgets = [self.Object('Widget 1'), self.Object('Widget 2')] app = stubs.FakeQApplication(all_widgets=widgets) From 21757a96a3bd73e81d0365731fedc726529676ca Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 08:25:46 -0500 Subject: [PATCH 031/161] Add docstrings to SqlCompletionModel overrides. Leaving QAbstractItemModel overrides undocumented made pylint angry. --- qutebrowser/completion/models/sqlmodel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index fcd8d37c6..2c7a25bc0 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -194,6 +194,7 @@ class SqlCompletionModel(QAbstractItemModel): return self.createIndex(row, 0, None) def rowCount(self, parent=QModelIndex()): + """Override QAbstractItemModel::rowCount.""" if not parent.isValid(): # top-level return len(self._categories) @@ -206,6 +207,7 @@ class SqlCompletionModel(QAbstractItemModel): return cat.rowCount() def columnCount(self, parent=QModelIndex()): + """Override QAbstractItemModel::columnCount.""" # pylint: disable=unused-argument return 3 From d658378af360765ff11fb416b6bf9f0f9ff160c5 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 17:04:10 -0500 Subject: [PATCH 032/161] Eliminate test interference from webkit history. Initializing the qtwebkit history backend left some global state that was leaking into other tests. --- tests/unit/browser/webkit/test_history.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 844c20fb1..43b30ed19 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -362,3 +362,6 @@ def test_init(backend, qapp, tmpdir, monkeypatch): assert default_interface is None objreg.delete('web-history') + if backend == usertypes.Backend.QtWebKit: + # prevent interference with future tests + QWebHistoryInterface.setDefaultInterface(None) From be38e181a86f1adb8464de6079979ffe45d9171c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 17:06:16 -0500 Subject: [PATCH 033/161] Install libqt5sql5-sqlite for debian CI. Needed for tests to pass with the new SQL dependency. --- scripts/dev/ci/travis_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index aa4c1c6c8..b200fb6d0 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -102,7 +102,7 @@ elif [[ $TRAVIS_OS_NAME == osx ]]; then exit 0 fi -pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit python3-pyqt5.qtsql" +pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit python3-pyqt5.qtsql libqt5sql5-sqlite" pip_install pip pip_install -r misc/requirements/requirements-tox.txt From 788babbb61f8b06f307a548622a2deea6af65bf9 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 20:32:03 -0500 Subject: [PATCH 034/161] Further prevent state leakage from test_init. test_history.test_init also leaked state by leaving the instantiated history as the parent of the QApp, which was causing test_debug to fail because it was trying to dump the history object left from test_history. --- tests/unit/browser/webkit/test_history.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 43b30ed19..890ff9a89 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -361,7 +361,8 @@ def test_init(backend, qapp, tmpdir, monkeypatch): # before (so we need to test webengine before webkit) assert default_interface is None + # prevent interference with future tests objreg.delete('web-history') + hist.setParent(None) if backend == usertypes.Backend.QtWebKit: - # prevent interference with future tests QWebHistoryInterface.setDefaultInterface(None) From fc5fd6096ab546a9bc1ad7208b710c48a2a7145b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 20:34:14 -0500 Subject: [PATCH 035/161] Revert "Initialize SQL for two failing tests." This reverts commit 386e227ce7534f1e427db7ba6d4e53dc153a49f3. The problem was really state leakage, initializing sql for these tests isn't necessary. --- tests/unit/browser/webkit/test_webkitelem.py | 2 +- tests/unit/utils/test_debug.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/browser/webkit/test_webkitelem.py b/tests/unit/browser/webkit/test_webkitelem.py index f6cd44a8c..deff844fe 100644 --- a/tests/unit/browser/webkit/test_webkitelem.py +++ b/tests/unit/browser/webkit/test_webkitelem.py @@ -211,7 +211,7 @@ class TestSelectorsAndFilters: assert self.TESTS @pytest.mark.parametrize('group, val, matching', TESTS) - def test_selectors(self, webframe, init_sql, group, val, matching): + def test_selectors(self, webframe, group, val, matching): webframe.setHtml('{}'.format(val)) # Make sure setting HTML succeeded and there's a new element assert len(webframe.findAllElements('*')) == 3 diff --git a/tests/unit/utils/test_debug.py b/tests/unit/utils/test_debug.py index 646858163..9b77b9628 100644 --- a/tests/unit/utils/test_debug.py +++ b/tests/unit/utils/test_debug.py @@ -244,7 +244,7 @@ class TestGetAllObjects: def __repr__(self): return '<{}>'.format(self._name) - def test_get_all_objects(self, stubs, monkeypatch, init_sql): + def test_get_all_objects(self, stubs, monkeypatch): # pylint: disable=unused-variable widgets = [self.Object('Widget 1'), self.Object('Widget 2')] app = stubs.FakeQApplication(all_widgets=widgets) From 6cc2095221e428b9f52b81a48043bce663771ddc Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 17 Feb 2017 08:13:36 -0500 Subject: [PATCH 036/161] Avoid keyconf circular import. The new function-based completion API introduced a circular import: config -> keyconf -> miscmodels -> config. config only depended on keyconf so it could initialize it as part of config.init. This can be resolved by moving this to keyconf.init and initializing keyconf as part of app.init. --- qutebrowser/app.py | 5 ++++ qutebrowser/config/config.py | 1 - qutebrowser/config/parsers/keyconf.py | 33 ++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 8b2aa7d69..44178bae3 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -45,6 +45,7 @@ import qutebrowser.resources from qutebrowser.completion.models import miscmodels from qutebrowser.commands import cmdutils, runners, cmdexc from qutebrowser.config import style, config, websettings, configexc +from qutebrowser.config.parsers import keyconf from qutebrowser.browser import (urlmarks, adblock, history, browsertab, downloads) from qutebrowser.browser.network import proxy @@ -423,6 +424,10 @@ def _init_modules(args, crash_handler): config.init(qApp) save_manager.init_autosave() + log.init.debug("Initializing keys...") + keyconf.init(qApp) + save_manager.init_autosave() + log.init.debug("Initializing sql...") sql.init() diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index dd71f5333..0b1893fc0 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -249,7 +249,6 @@ def init(parent=None): parent: The parent to pass to QObjects which get initialized. """ _init_main_config(parent) - _init_key_config(parent) _init_misc() diff --git a/qutebrowser/config/parsers/keyconf.py b/qutebrowser/config/parsers/keyconf.py index 5a780a786..751eafb71 100644 --- a/qutebrowser/config/parsers/keyconf.py +++ b/qutebrowser/config/parsers/keyconf.py @@ -22,15 +22,46 @@ import collections import os.path import itertools +import sys from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.config import configdata, textwrapper from qutebrowser.commands import cmdutils, cmdexc -from qutebrowser.utils import log, utils, qtutils, message, usertypes +from qutebrowser.utils import (log, utils, qtutils, message, usertypes, objreg, + standarddir, error) from qutebrowser.completion.models import miscmodels +def init(parent=None): + """Read and save keybindings. + + Args: + parent: The parent to use for the KeyConfigParser. + """ + args = objreg.get('args') + try: + key_config = KeyConfigParser(standarddir.config(), 'keys.conf', + args.relaxed_config, parent=parent) + except (KeyConfigError, UnicodeDecodeError) as e: + log.init.exception(e) + errstr = "Error while reading key config:\n" + if e.lineno is not None: + errstr += "In line {}: ".format(e.lineno) + error.handle_fatal_exc(e, args, "Error while reading key config!", + pre_text=errstr) + # We didn't really initialize much so far, so we just quit hard. + sys.exit(usertypes.Exit.err_key_config) + else: + objreg.register('key-config', key_config) + save_manager = objreg.get('save-manager') + filename = os.path.join(standarddir.config(), 'keys.conf') + save_manager.add_saveable( + 'key-config', key_config.save, key_config.config_dirty, + config_opt=('general', 'auto-save-config'), filename=filename, + dirty=key_config.is_dirty) + + class KeyConfigError(Exception): """Raised on errors with the key config. From 6a8b1d51fa7326ac8080217c0bcfaf3d78c53cd7 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 19 Feb 2017 20:58:10 -0500 Subject: [PATCH 037/161] Avoid config -> configmodel circular import. Avoid the config dependency by using objreg.get('config') instead of config.get. --- qutebrowser/completion/models/configmodel.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index 406d8d572..7c7e3a0c6 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -19,8 +19,9 @@ """Functions that return config-related completion models.""" -from qutebrowser.config import config, configdata, configexc +from qutebrowser.config import configdata, configexc from qutebrowser.completion.models import base +from qutebrowser.utils import objreg def section(): @@ -54,6 +55,7 @@ def option(sectname): desc = "" else: desc = desc.splitlines()[0] + config = objreg.get('config') val = config.get(sectname, name, raw=True) model.new_item(cat, name, desc, val) return model @@ -68,6 +70,7 @@ def value(sectname, optname): """ model = base.CompletionModel(column_widths=(20, 70, 10)) cur_cat = model.new_category("Current/Default") + config = objreg.get('config') try: val = config.get(sectname, optname, raw=True) or '""' except (configexc.NoSectionError, configexc.NoOptionError): From 1474e38eec4ca91dd00f5b7bce60eaf86e0048b0 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 19 Feb 2017 21:36:02 -0500 Subject: [PATCH 038/161] Add urlmodel to perfect_files --- scripts/dev/check_coverage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 7e5a2d47a..3b1611a1c 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -156,6 +156,8 @@ PERFECT_FILES = [ ('tests/unit/completion/test_models.py', 'completion/models/base.py'), + ('tests/unit/completion/test_models.py', + 'completion/models/urlmodel.py'), ('tests/unit/completion/test_sortfilter.py', 'completion/models/sortfilter.py'), From c6645d47bab522a9aa9869fe19fb9ff93e697c01 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 19 Feb 2017 21:39:15 -0500 Subject: [PATCH 039/161] Remove newest_slice and StatusBar._option. newest_slice is no longer needed after the completion refactor. Now that history is based on the SQL backend, LIMIT is used instead. StatusBar._option is not used, though I'm not sure why vulture only caught it now. --- qutebrowser/utils/utils.py | 19 ------------------- tests/unit/utils/test_utils.py | 31 ------------------------------- 2 files changed, 50 deletions(-) diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py index 0b527637e..9d1320995 100644 --- a/qutebrowser/utils/utils.py +++ b/qutebrowser/utils/utils.py @@ -737,25 +737,6 @@ def sanitize_filename(name, replacement='_'): return name -def newest_slice(iterable, count): - """Get an iterable for the n newest items of the given iterable. - - Args: - count: How many elements to get. - 0: get no items: - n: get the n newest items - -1: get all items - """ - if count < -1: - raise ValueError("count can't be smaller than -1!") - elif count == 0: - return [] - elif count == -1 or len(iterable) < count: - return iterable - else: - return itertools.islice(iterable, len(iterable) - count, len(iterable)) - - def set_clipboard(data, selection=False): """Set the clipboard to some given data.""" if selection and not supports_selection(): diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index 4b03dcf8c..47595ebf5 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -759,37 +759,6 @@ def test_sanitize_filename_empty_replacement(): assert utils.sanitize_filename(name, replacement=None) == 'Bad File' -class TestNewestSlice: - - """Test newest_slice.""" - - def test_count_minus_two(self): - """Test with a count of -2.""" - with pytest.raises(ValueError): - utils.newest_slice([], -2) - - @pytest.mark.parametrize('items, count, expected', [ - # Count of -1 (all elements). - (range(20), -1, range(20)), - # Count of 0 (no elements). - (range(20), 0, []), - # Count which is much smaller than the iterable. - (range(20), 5, [15, 16, 17, 18, 19]), - # Count which is exactly one smaller.""" - (range(5), 4, [1, 2, 3, 4]), - # Count which is just as large as the iterable.""" - (range(5), 5, range(5)), - # Count which is one bigger than the iterable. - (range(5), 6, range(5)), - # Count which is much bigger than the iterable. - (range(5), 50, range(5)), - ]) - def test_good(self, items, count, expected): - """Test slices which shouldn't raise an exception.""" - sliced = utils.newest_slice(items, count) - assert list(sliced) == list(expected) - - class TestGetSetClipboard: @pytest.fixture(autouse=True) From 496859007595874e8f93a1b14ce641460531f4b6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 20 Feb 2017 08:17:21 -0500 Subject: [PATCH 040/161] Mention needed SQL packages in INSTALL. With the new SQL completion backend, some distros must install sql packages. This also removes trailing whitespace from one line. --- INSTALL.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.asciidoc b/INSTALL.asciidoc index ad4b8446f..48427e121 100644 --- a/INSTALL.asciidoc +++ b/INSTALL.asciidoc @@ -27,7 +27,7 @@ Using the packages Install the dependencies via apt-get: ---- -# apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-jinja2 python3-pygments python3-yaml +# apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-jinja2 python3-pygments python3-yaml python3-pyqt5.qtsql libqt5sql5-sqlite ---- On Debian Stretch or Ubuntu 17.04 or later, it's also recommended to use the @@ -53,7 +53,7 @@ Build it from git Install the dependencies via apt-get: ---- -# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python-tox python3-sip python3-dev +# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python-tox python3-sip python3-dev python3-pyqt5.qtsql libqt5sql5-sqlite ---- On Debian Stretch or Ubuntu 17.04 or later, it's also recommended to install From a050cb94f675779986a683a3216890ab21bf3803 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 20 Feb 2017 08:33:38 -0500 Subject: [PATCH 041/161] Report sqlite version with --version. --- qutebrowser/misc/sql.py | 7 +++++++ qutebrowser/utils/version.py | 6 +++++- tests/unit/misc/test_sql.py | 3 +++ tests/unit/utils/test_version.py | 2 ++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index c051acf03..e0b55dbd2 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -41,6 +41,13 @@ def close(): QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) +def version(): + """Return the sqlite version string.""" + result = run_query("select sqlite_version()") + result.next() + return result.record().value(0) + + def run_query(querystr, values=None): """Run the given SQL query string on the database. diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index 08779e097..ac74f997a 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -45,7 +45,7 @@ except ImportError: # pragma: no cover import qutebrowser from qutebrowser.utils import log, utils, standarddir, usertypes, qtutils -from qutebrowser.misc import objects, earlyinit +from qutebrowser.misc import objects, earlyinit, sql from qutebrowser.browser import pdfjs @@ -327,6 +327,10 @@ def version(): lines += ['pdf.js: {}'.format(_pdfjs_version())] + sql.init() + lines += ['sqlite: {}'.format(sql.version())] + sql.close() + lines += [ 'SSL: {}'.format(QSslSocket.sslLibraryVersionString()), '', diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 4a5299b48..b24f2fbcf 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -120,3 +120,6 @@ def test_delete_all(qtbot): with qtbot.waitSignal(table.changed): table.delete_all() assert list(table) == [] + +def test_version(): + assert isinstance(sql.version(), str) diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 54ee2cdd0..8c0f16081 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -834,6 +834,7 @@ def test_version_output(git_commit, frozen, style, with_webkit, 'QApplication': (stubs.FakeQApplication(style='STYLE') if style else stubs.FakeQApplication(instance=None)), 'QLibraryInfo.location': (lambda _loc: 'QT PATH'), + 'sql.version': lambda: 'SQLITE VERSION', } substitutions = { @@ -892,6 +893,7 @@ def test_version_output(git_commit, frozen, style, with_webkit, MODULE VERSION 1 MODULE VERSION 2 pdf.js: PDFJS VERSION + sqlite: SQLITE VERSION SSL: SSL VERSION {style} Platform: PLATFORM, ARCHITECTURE{linuxdist} From 3c676f9562606bcddaa2291531ff488c44880b9b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 20 Feb 2017 12:15:06 -0500 Subject: [PATCH 042/161] Fix pylint/flake8 errors for SQL work. --- qutebrowser/utils/utils.py | 1 - tests/unit/misc/test_sql.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py index 9d1320995..a2c9c104d 100644 --- a/qutebrowser/utils/utils.py +++ b/qutebrowser/utils/utils.py @@ -28,7 +28,6 @@ import os.path import collections import functools import contextlib -import itertools import socket import shlex diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index b24f2fbcf..8bf7ddb4b 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -121,5 +121,6 @@ def test_delete_all(qtbot): table.delete_all() assert list(table) == [] + def test_version(): assert isinstance(sql.version(), str) From 56f3b3a02709736ba702b3b3ef4342e3a4c98a42 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 21 Feb 2017 08:27:52 -0500 Subject: [PATCH 043/161] Small review fixups for SQL implementation. Respond to the low-hanging code review fruit: - Clean up some comments - Remove an acidentally added duplicate init_autosave - Combine two test_history tests - Move test_init cleanup into a fixture to ensure it gets called. - Name the _ argument of bind(_) to _key - Ensure index is valid for first_item/last_item - Move SqlException to top of module - Rename test_index to test_getitem - Return QItemFlags.None instead of None - Fix copyright dates (its 2017 now!) - Use * to force some args to be keyword-only - Make some returns explicit - Add sql to LOGGER_NAMES - Add a comment to explain the sql escape statement --- qutebrowser/app.py | 1 - qutebrowser/browser/urlmarks.py | 12 ++---- qutebrowser/completion/models/base.py | 3 +- qutebrowser/completion/models/miscmodels.py | 4 +- qutebrowser/completion/models/sqlmodel.py | 34 +++++++++------- qutebrowser/misc/sql.py | 19 ++++----- qutebrowser/utils/log.py | 2 +- tests/helpers/fixtures.py | 2 +- tests/unit/browser/webkit/test_history.py | 44 +++++++++++---------- tests/unit/completion/test_sqlmodel.py | 2 +- tests/unit/misc/test_sql.py | 4 +- 11 files changed, 65 insertions(+), 62 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 44178bae3..33726b058 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -426,7 +426,6 @@ def _init_modules(args, crash_handler): log.init.debug("Initializing keys...") keyconf.init(qApp) - save_manager.init_autosave() log.init.debug("Initializing sql...") sql.init() diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index 5f7a4fb77..580d77881 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -104,10 +104,7 @@ class UrlMarkManager(sql.SqlTable): class QuickmarkManager(UrlMarkManager): - """Manager for quickmarks. - - The primary key for quickmarks is their *name*. - """ + """Manager for quickmarks.""" def __init__(self, parent=None): super().__init__('Quickmarks', ['name', 'url'], primary_key='name', @@ -179,7 +176,7 @@ class QuickmarkManager(UrlMarkManager): set_mark() def delete_by_qurl(self, url): - """Look up a quickmark by QUrl, returning its name.""" + """Delete a quickmark by QUrl (as opposed to by name).""" qtutils.ensure_valid(url) urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) try: @@ -204,10 +201,7 @@ class QuickmarkManager(UrlMarkManager): class BookmarkManager(UrlMarkManager): - """Manager for bookmarks. - - The primary key for bookmarks is their *url*. - """ + """Manager for bookmarks.""" def __init__(self, parent=None): super().__init__('Bookmarks', ['url', 'title'], primary_key='url', diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py index 112c6ed00..f5148ff9c 100644 --- a/qutebrowser/completion/models/base.py +++ b/qutebrowser/completion/models/base.py @@ -36,9 +36,10 @@ class CompletionModel(QStandardItemModel): Attributes: column_widths: The width percentages of the columns used in the + completion view. """ - def __init__(self, column_widths=(30, 70, 0), columns_to_filter=None, + def __init__(self, *, column_widths=(30, 70, 0), columns_to_filter=None, delete_cur_item=None, parent=None): super().__init__(parent) self.setColumnCount(3) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 8bb234521..f5ed72f5d 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -130,11 +130,11 @@ def buffer(): return model -def bind(_): +def bind(_key): """A CompletionModel filled with all bindable commands and descriptions. Args: - _: the key being bound. + _key: the key being bound. """ # TODO: offer a 'Current binding' completion based on the key. model = base.CompletionModel(column_widths=(20, 60, 20)) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 2c7a25bc0..1053fb2f2 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2016 Ryan Roden-Corrent (rcorre) +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # @@ -24,12 +24,14 @@ import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from PyQt5.QtSql import QSqlQueryModel -from qutebrowser.utils import log +from qutebrowser.utils import log, qtutils from qutebrowser.misc import sql class _SqlCompletionCategory(QSqlQueryModel): - def __init__(self, name, sort_by, sort_order, limit, select, where, + """Wraps a SqlQuery for use as a completion category.""" + + def __init__(self, name, *, sort_by, sort_order, limit, select, where, columns_to_filter, parent=None): super().__init__(parent=parent) self.tablename = name @@ -38,6 +40,8 @@ class _SqlCompletionCategory(QSqlQueryModel): self._fields = [query.record().fieldName(i) for i in columns_to_filter] querystr = 'select {} from {} where ('.format(select, name) + # the incoming pattern will have literal % and _ escaped with '\' + # we need to tell sql to treat '\' as an escape character querystr += ' or '.join("{} like ? escape '\\'".format(f) for f in self._fields) querystr += ')' @@ -45,6 +49,7 @@ class _SqlCompletionCategory(QSqlQueryModel): querystr += ' and ' + where if sort_by: + assert sort_order is not None sortstr = 'asc' if sort_order == Qt.AscendingOrder else 'desc' querystr += ' order by {} {}'.format(sort_by, sortstr) @@ -69,19 +74,15 @@ class SqlCompletionModel(QAbstractItemModel): Top level indices represent categories, each of which is backed by a single table. Child indices represent rows of those tables. - Class Attributes: - COLUMN_WIDTHS: The width percentages of the columns used in the - completion view. - Attributes: column_widths: The width percentages of the columns used in the - completion view. + completion view. columns_to_filter: A list of indices of columns to apply the filter to. pattern: Current filter pattern, used for highlighting. _categories: The category tables. """ - def __init__(self, column_widths=(30, 70, 0), columns_to_filter=None, + def __init__(self, *, column_widths=(30, 70, 0), columns_to_filter=None, delete_cur_item=None, parent=None): super().__init__(parent) self.columns_to_filter = columns_to_filter or [0] @@ -103,8 +104,9 @@ class SqlCompletionModel(QAbstractItemModel): # categories have an empty internalPointer if index.isValid() and not index.internalPointer(): return self._categories[index.row()] + return None - def new_category(self, name, select='*', where=None, sort_by=None, + def new_category(self, name, *, select='*', where=None, sort_by=None, sort_order=None, limit=None): """Create a new completion category and add it to this model. @@ -135,7 +137,7 @@ class SqlCompletionModel(QAbstractItemModel): Return: The item data, or None on an invalid index. """ if not index.isValid() or role != Qt.DisplayRole: - return + return None if not index.parent().isValid(): if index.column() == 0: return self._categories[index.row()].tablename @@ -152,7 +154,7 @@ class SqlCompletionModel(QAbstractItemModel): Return: The item flags, or Qt.NoItemFlags on error. """ if not index.isValid(): - return + return Qt.NoItemFlags if index.parent().isValid(): # item return (Qt.ItemIsEnabled | Qt.ItemIsSelectable | @@ -253,7 +255,9 @@ class SqlCompletionModel(QAbstractItemModel): for row, table in enumerate(self._categories): if table.rowCount() > 0: parent = self.index(row, 0) - return self.index(0, 0, parent) + index = self.index(0, 0, parent) + qtutils.ensure_valid(index) + return index return QModelIndex() def last_item(self): @@ -262,7 +266,9 @@ class SqlCompletionModel(QAbstractItemModel): childcount = table.rowCount() if childcount > 0: parent = self.index(row, 0) - return self.index(childcount - 1, 0, parent) + index = self.index(childcount - 1, 0, parent) + qtutils.ensure_valid(index) + return index return QModelIndex() diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index e0b55dbd2..c9249d362 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2016 Ryan Roden-Corrent (rcorre) +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # @@ -27,13 +27,21 @@ from qutebrowser.utils import log import collections +class SqlException(Exception): + + """Raised on an error interacting with the SQL database.""" + + pass + + def init(): """Initialize the SQL database connection.""" database = QSqlDatabase.addDatabase('QSQLITE') # In-memory database, see https://sqlite.org/inmemorydb.html database.setDatabaseName(':memory:') if not database.open(): - raise SqlException("Failed to open in-memory sqlite database") + raise SqlException("Failed to open in-memory sqlite database: {}" + .format(database.lastError().text())) def close(): @@ -169,10 +177,3 @@ class SqlTable(QObject): """Remove all row from the table.""" run_query("DELETE FROM {}".format(self._name)) self.changed.emit() - - -class SqlException(Exception): - - """Raised on an error interacting with the SQL database.""" - - pass diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py index 513c05062..6cdc61f41 100644 --- a/qutebrowser/utils/log.py +++ b/qutebrowser/utils/log.py @@ -94,7 +94,7 @@ LOGGER_NAMES = [ 'commands', 'signals', 'downloads', 'js', 'qt', 'rfc6266', 'ipc', 'shlexer', 'save', 'message', 'config', 'sessions', - 'webelem', 'prompt', 'network' + 'webelem', 'prompt', 'network', 'sql' ] diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 7fc05a578..0b69ad89f 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -457,7 +457,7 @@ def short_tmpdir(): yield py.path.local(tdir) # pylint: disable=no-member -@pytest.fixture() +@pytest.fixture def init_sql(): """Initialize the SQL module, and shut it down after the test.""" sql.init() diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 890ff9a89..9d8711152 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -193,21 +193,15 @@ def test_clear(qtbot, tmpdir): assert lines == ['67890 http://www.the-compiler.org/'] -def test_add_item(qtbot, hist): +@pytest.mark.parametrize('item', [ + ('http://www.example.com', 12346, 'the title', False), + ('http://www.example.com', 12346, 'the title', True) +]) +def test_add_item(qtbot, hist, item): list(hist.async_read()) - url = 'http://www.example.com/' - - hist.add_url(QUrl(url), atime=12345, title="the title") - - assert hist[url] == (url, 'the title', 12345, False) - - -def test_add_item_redirect(qtbot, hist): - list(hist.async_read()) - url = 'http://www.example.com/' - hist.add_url(QUrl(url), redirect=True, atime=12345) - - assert hist[url] == (url, '', 12345, True) + (url, atime, title, redirect) = item + hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) + assert hist[url] == (url, title, atime, redirect) def test_add_item_redirect_update(qtbot, tmpdir, fake_save_manager): @@ -329,9 +323,23 @@ def test_history_interface(qtbot, webview, hist_interface): webview.load(url) +@pytest.fixture +def cleanup_init(): + yield + # prevent test_init from leaking state + hist = objreg.get('web-history') + hist.setParent(None) + objreg.delete('web-history') + try: + from PyQt5.QtWebKit import QWebHistoryInterface + QWebHistoryInterface.setDefaultInterface(None) + except: + pass + + @pytest.mark.parametrize('backend', [usertypes.Backend.QtWebEngine, usertypes.Backend.QtWebKit]) -def test_init(backend, qapp, tmpdir, monkeypatch): +def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): if backend == usertypes.Backend.QtWebKit: pytest.importorskip('PyQt5.QtWebKitWidgets') else: @@ -360,9 +368,3 @@ def test_init(backend, qapp, tmpdir, monkeypatch): # For this to work, nothing can ever have called setDefaultInterface # before (so we need to test webengine before webkit) assert default_interface is None - - # prevent interference with future tests - objreg.delete('web-history') - hist.setParent(None) - if backend == usertypes.Backend.QtWebKit: - QWebHistoryInterface.setDefaultInterface(None) diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index f6daa722c..044eecb35 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2016 Ryan Roden-Corrent (rcorre) +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 8bf7ddb4b..f4898b25d 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2016 Ryan Roden-Corrent (rcorre) +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # @@ -102,7 +102,7 @@ def test_contains(): assert 'thirteen' in table -def test_index(): +def test_getitem(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') table.insert(['one', 1, False]) table.insert(['nine', 9, False]) From c4c5723a61f170cb5945b49ce18b46c5b477ff42 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 21 Feb 2017 09:05:22 -0500 Subject: [PATCH 044/161] Sort history completion entries by atime. --- qutebrowser/completion/models/sqlmodel.py | 7 +++---- qutebrowser/completion/models/urlmodel.py | 2 +- tests/unit/completion/test_sqlmodel.py | 14 +++++++------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 1053fb2f2..99c47cf28 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -49,9 +49,8 @@ class _SqlCompletionCategory(QSqlQueryModel): querystr += ' and ' + where if sort_by: - assert sort_order is not None - sortstr = 'asc' if sort_order == Qt.AscendingOrder else 'desc' - querystr += ' order by {} {}'.format(sort_by, sortstr) + assert sort_order == 'asc' or sort_order == 'desc' + querystr += ' order by {} {}'.format(sort_by, sort_order) if limit: querystr += ' limit {}'.format(limit) @@ -115,7 +114,7 @@ class SqlCompletionModel(QAbstractItemModel): select: A custom result column expression for the select statement. where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. - sort_order: Sorting order, if sort_by is non-None. + sort_order: Either 'asc' or 'desc', if sort_by is non-None limit: Maximum row count to return on a query. Return: A new CompletionCategory. diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 387603dd5..6a10d9c44 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -70,7 +70,7 @@ def url(): model.new_category('Quickmarks', select='url, name') model.new_category('Bookmarks') model.new_category('History', - limit=limit, + limit=limit, sort_order='desc', sort_by='atime', select='url, title, {}'.format(select_time), where='not redirect') return model diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 044eecb35..ab14bce75 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -73,31 +73,31 @@ def test_count(rowcounts, expected): @pytest.mark.parametrize('sort_by, sort_order, data, expected', [ - (None, Qt.AscendingOrder, + (None, 'asc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), - ('a', Qt.AscendingOrder, + ('a', 'asc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - ('a', Qt.DescendingOrder, + ('a', 'desc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - ('b', Qt.AscendingOrder, + ('b', 'asc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - ('b', Qt.DescendingOrder, + ('b', 'desc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - ('c', Qt.AscendingOrder, + ('c', 'asc', [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], [('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]), - ('c', Qt.DescendingOrder, + ('c', 'desc', [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), ]) From 99c9b2d3963701fd598a7d036096c73183f778c6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 22 Feb 2017 21:05:03 -0500 Subject: [PATCH 045/161] Fix two small mistakes after SQL code review. urlmodel is now sorted, so the test had to be adjusted. Also remove one unused import. --- tests/unit/completion/test_models.py | 2 +- tests/unit/completion/test_sqlmodel.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index ddc746aa6..b866e6439 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -297,7 +297,7 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ], "History": [ - ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ('https://github.com', 'https://github.com', '2016-05-01'), ('https://python.org', 'Welcome to Python.org', '2016-03-08'), ], }) diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index ab14bce75..49dc96ccf 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -20,7 +20,6 @@ """Tests for the base sql completion model.""" import pytest -from PyQt5.QtCore import Qt from qutebrowser.misc import sql from qutebrowser.completion.models import sqlmodel From 921211bbaa09904fbeddf882cbe5f4d1ed1bb1ec Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 22 Feb 2017 21:12:23 -0500 Subject: [PATCH 046/161] Remove web-history-max-items. This was a performance optimization that shouldn't be needed with the new SQL history backend. This also removes support for the LIMIT feature from SqlTable as it only existed to support web-history-max-items. --- qutebrowser/completion/models/sqlmodel.py | 10 +++------- qutebrowser/completion/models/urlmodel.py | 3 +-- qutebrowser/config/config.py | 1 + qutebrowser/config/configdata.py | 5 ----- tests/unit/completion/test_models.py | 14 +++++--------- tests/unit/completion/test_sqlmodel.py | 9 --------- 6 files changed, 10 insertions(+), 32 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 99c47cf28..e3e5a5c5f 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -31,7 +31,7 @@ from qutebrowser.misc import sql class _SqlCompletionCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" - def __init__(self, name, *, sort_by, sort_order, limit, select, where, + def __init__(self, name, *, sort_by, sort_order, select, where, columns_to_filter, parent=None): super().__init__(parent=parent) self.tablename = name @@ -52,9 +52,6 @@ class _SqlCompletionCategory(QSqlQueryModel): assert sort_order == 'asc' or sort_order == 'desc' querystr += ' order by {} {}'.format(sort_by, sort_order) - if limit: - querystr += ' limit {}'.format(limit) - self._querystr = querystr self.set_pattern('%') @@ -106,7 +103,7 @@ class SqlCompletionModel(QAbstractItemModel): return None def new_category(self, name, *, select='*', where=None, sort_by=None, - sort_order=None, limit=None): + sort_order=None): """Create a new completion category and add it to this model. Args: @@ -115,12 +112,11 @@ class SqlCompletionModel(QAbstractItemModel): where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Either 'asc' or 'desc', if sort_by is non-None - limit: Maximum row count to return on a query. Return: A new CompletionCategory. """ cat = _SqlCompletionCategory(name, parent=self, sort_by=sort_by, - sort_order=sort_order, limit=limit, + sort_order=sort_order, select=select, where=where, columns_to_filter=self.columns_to_filter) self._categories.append(cat) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 6a10d9c44..0b357c3d7 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -64,13 +64,12 @@ def url(): model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), columns_to_filter=[_URLCOL, _TEXTCOL], delete_cur_item=_delete_url) - limit = config.get('completion', 'web-history-max-items') timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) model.new_category('Quickmarks', select='url, name') model.new_category('Bookmarks') model.new_category('History', - limit=limit, sort_order='desc', sort_by='atime', + sort_order='desc', sort_by='atime', select='url, title, {}'.format(select_time), where='not redirect') return model diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index 0b1893fc0..e7e74cc19 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -414,6 +414,7 @@ class ConfigManager(QObject): ('storage', 'offline-storage-default-quota'), ('storage', 'offline-web-application-cache-quota'), ('content', 'css-regions'), + ('completion', 'web-history-max-items'), ] CHANGED_OPTIONS = { ('content', 'cookies-accept'): diff --git a/qutebrowser/config/configdata.py b/qutebrowser/config/configdata.py index 28cf41ac5..fd3235c8b 100644 --- a/qutebrowser/config/configdata.py +++ b/qutebrowser/config/configdata.py @@ -506,11 +506,6 @@ def data(readonly=False): "How many commands to save in the command history.\n\n" "0: no history / -1: unlimited"), - ('web-history-max-items', - SettingValue(typ.Int(minval=-1), '1000'), - "How many URLs to show in the web history.\n\n" - "0: no history / -1: unlimited"), - ('quick-complete', SettingValue(typ.Bool(), 'true'), "Whether to move on to the next part when there's only one " diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index b866e6439..224d7ed59 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -274,13 +274,10 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, Verify that: - quickmarks, bookmarks, and urls are included - - no more than 'web-history-max-items' items are included (TODO) - - the most recent entries are included + - entries are sorted by access time - redirect entries are not included """ - # TODO: time formatting and item limiting - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - 'web-history-max-items': 2} + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -299,6 +296,7 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, "History": [ ('https://github.com', 'https://github.com', '2016-05-01'), ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), ], }) @@ -307,8 +305,7 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): """Test deleting a bookmark from the url completion model.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - 'web-history-max-items': 2} + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -325,8 +322,7 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): """Test deleting a bookmark from the url completion model.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - 'web-history-max-items': 2} + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 49dc96ccf..8b9727dfd 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -206,15 +206,6 @@ def test_first_last_item(data, first, last): assert model.data(model.last_item()) == last -def test_limit(): - table = sql.SqlTable('test_limit', ['a'], primary_key='a') - for i in range(5): - table.insert([i]) - model = sqlmodel.SqlCompletionModel() - model.new_category('test_limit', limit=3) - assert model.count() == 3 - - def test_select(): table = sql.SqlTable('test_select', ['a', 'b', 'c'], primary_key='a') table.insert(['foo', 'bar', 'baz']) From e3a33ca42744b374f01cfd8792fdac845e3b22e0 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 22 Feb 2017 22:25:11 -0500 Subject: [PATCH 047/161] Implement a hybrid list/sql completion model. Now all completion models are of a single type called CompletionModel. This model combines one or more categories. A category can either be a ListCategory or a SqlCategory. This simplifies the API, and will allow the use of models that combine simple list-based and sql sources. This is important for two reasons: - Adding searchengines to url completion - Using an on-disk sqlite database for history, while keeping bookmarks and quickmars as text files. --- qutebrowser/completion/completer.py | 24 +-- qutebrowser/completion/completiondelegate.py | 2 +- qutebrowser/completion/completionwidget.py | 6 +- qutebrowser/completion/models/base.py | 105 ------------- .../{sqlmodel.py => completionmodel.py} | 126 ++++++--------- qutebrowser/completion/models/configmodel.py | 37 ++--- qutebrowser/completion/models/listcategory.py | 66 ++++++++ qutebrowser/completion/models/miscmodels.py | 58 ++++--- qutebrowser/completion/models/sortfilter.py | 43 +----- qutebrowser/completion/models/sqlcategory.py | 64 ++++++++ qutebrowser/completion/models/urlmodel.py | 16 +- tests/unit/completion/test_completionmodel.py | 146 ++++++++++++++++++ .../unit/completion/test_completionwidget.py | 52 +++---- tests/unit/completion/test_models.py | 30 ++-- tests/unit/completion/test_sortfilter.py | 58 +------ 15 files changed, 426 insertions(+), 407 deletions(-) delete mode 100644 qutebrowser/completion/models/base.py rename qutebrowser/completion/models/{sqlmodel.py => completionmodel.py} (63%) create mode 100644 qutebrowser/completion/models/listcategory.py create mode 100644 qutebrowser/completion/models/sqlcategory.py create mode 100644 tests/unit/completion/test_completionmodel.py diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index f403354c3..94c7f9352 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -24,7 +24,7 @@ from PyQt5.QtCore import pyqtSlot, QObject, QTimer from qutebrowser.config import config from qutebrowser.commands import cmdutils, runners from qutebrowser.utils import log, utils -from qutebrowser.completion.models import sortfilter, miscmodels +from qutebrowser.completion.models import miscmodels class Completer(QObject): @@ -62,22 +62,6 @@ class Completer(QObject): completion = self.parent() return completion.model() - def _get_completion_model(self, completion, pos_args): - """Get a completion model based on an enum member. - - Args: - completion: A usertypes.Completion member. - pos_args: The positional args entered before the cursor. - - Return: - A completion model or None. - """ - model = completion(*pos_args) - if model is None or hasattr(model, 'set_pattern'): - return model - else: - return sortfilter.CompletionFilterModel(source=model, parent=self) - def _get_new_completion(self, before_cursor, under_cursor): """Get a new completion. @@ -96,9 +80,8 @@ class Completer(QObject): log.completion.debug("After removing flags: {}".format(before_cursor)) if not before_cursor: # '|' or 'set|' - model = miscmodels.command() log.completion.debug('Starting command completion') - return sortfilter.CompletionFilterModel(source=model, parent=self) + return miscmodels.command() try: cmd = cmdutils.cmd_dict[before_cursor[0]] except KeyError: @@ -113,7 +96,8 @@ class Completer(QObject): return None if completion is None: return None - model = self._get_completion_model(completion, before_cursor[1:]) + + model = completion(*before_cursor[1:]) log.completion.debug('Starting {} completion' .format(completion.__name__)) return model diff --git a/qutebrowser/completion/completiondelegate.py b/qutebrowser/completion/completiondelegate.py index 1d5dfadf0..776b2164c 100644 --- a/qutebrowser/completion/completiondelegate.py +++ b/qutebrowser/completion/completiondelegate.py @@ -197,7 +197,7 @@ class CompletionItemDelegate(QStyledItemDelegate): if index.parent().isValid(): pattern = index.model().pattern - columns_to_filter = index.model().srcmodel.columns_to_filter + columns_to_filter = index.model().columns_to_filter if index.column() in columns_to_filter and pattern: repl = r'\g<0>' text = re.sub(re.escape(pattern).replace(r'\ ', r'|'), diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 3d2aef27e..f6a980612 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -299,7 +299,7 @@ class CompletionView(QTreeView): if pattern is not None: model.set_pattern(pattern) - self._column_widths = model.srcmodel.column_widths + self._column_widths = model.column_widths self._resize_columns() self._maybe_update_geometry() @@ -368,7 +368,7 @@ class CompletionView(QTreeView): """Delete the current completion item.""" if not self.currentIndex().isValid(): raise cmdexc.CommandError("No item selected!") - if self.model().srcmodel.delete_cur_item is None: + if self.model().delete_cur_item is None: raise cmdexc.CommandError("Cannot delete this item.") else: - self.model().srcmodel.delete_cur_item(self) + self.model().delete_cur_item(self) diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py deleted file mode 100644 index f5148ff9c..000000000 --- a/qutebrowser/completion/models/base.py +++ /dev/null @@ -1,105 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2014-2017 Florian Bruhin (The Compiler) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""The base completion model for completion in the command line. - -Module attributes: - Role: An enum of user defined model roles. -""" - -from PyQt5.QtCore import Qt -from PyQt5.QtGui import QStandardItemModel, QStandardItem - - -class CompletionModel(QStandardItemModel): - - """A simple QStandardItemModel adopted for completions. - - Used for showing completions later in the CompletionView. Supports setting - marks and adding new categories/items easily. - - Attributes: - column_widths: The width percentages of the columns used in the - completion view. - """ - - def __init__(self, *, column_widths=(30, 70, 0), columns_to_filter=None, - delete_cur_item=None, parent=None): - super().__init__(parent) - self.setColumnCount(3) - self.columns_to_filter = columns_to_filter or [0] - self.column_widths = column_widths - self.delete_cur_item = delete_cur_item - - def new_category(self, name): - """Add a new category to the model. - - Args: - name: The name of the category to add. - - Return: - The created QStandardItem. - """ - cat = QStandardItem(name) - self.appendRow(cat) - return cat - - def new_item(self, cat, name, desc='', misc=None): - """Add a new item to a category. - - Args: - cat: The parent category. - name: The name of the item. - desc: The description of the item. - misc: Misc text to display. - - Return: - A (nameitem, descitem, miscitem) tuple. - """ - assert not isinstance(name, int) - assert not isinstance(desc, int) - assert not isinstance(misc, int) - - nameitem = QStandardItem(name) - descitem = QStandardItem(desc) - miscitem = QStandardItem(misc) - - cat.appendRow([nameitem, descitem, miscitem]) - - def flags(self, index): - """Return the item flags for index. - - Override QAbstractItemModel::flags. - - Args: - index: The QModelIndex to get item flags for. - - Return: - The item flags, or Qt.NoItemFlags on error. - """ - if not index.isValid(): - return - - if index.parent().isValid(): - # item - return (Qt.ItemIsEnabled | Qt.ItemIsSelectable | - Qt.ItemNeverHasChildren) - else: - # category - return Qt.NoItemFlags diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/completionmodel.py similarity index 63% rename from qutebrowser/completion/models/sqlmodel.py rename to qutebrowser/completion/models/completionmodel.py index e3e5a5c5f..b8dcc4601 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) +# Copyright 2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # @@ -17,65 +17,29 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""A completion model backed by SQL tables.""" +"""A model that proxies access to one or more completion categories.""" import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel -from PyQt5.QtSql import QSqlQueryModel from qutebrowser.utils import log, qtutils -from qutebrowser.misc import sql +from qutebrowser.completion.models import sortfilter, listcategory, sqlcategory -class _SqlCompletionCategory(QSqlQueryModel): - """Wraps a SqlQuery for use as a completion category.""" +class CompletionModel(QAbstractItemModel): - def __init__(self, name, *, sort_by, sort_order, select, where, - columns_to_filter, parent=None): - super().__init__(parent=parent) - self.tablename = name + """A model that proxies access to one or more completion categories. - query = sql.run_query('select * from {} limit 1'.format(name)) - self._fields = [query.record().fieldName(i) for i in columns_to_filter] - - querystr = 'select {} from {} where ('.format(select, name) - # the incoming pattern will have literal % and _ escaped with '\' - # we need to tell sql to treat '\' as an escape character - querystr += ' or '.join("{} like ? escape '\\'".format(f) - for f in self._fields) - querystr += ')' - if where: - querystr += ' and ' + where - - if sort_by: - assert sort_order == 'asc' or sort_order == 'desc' - querystr += ' order by {} {}'.format(sort_by, sort_order) - - self._querystr = querystr - self.set_pattern('%') - - def set_pattern(self, pattern): - query = sql.run_query(self._querystr, [pattern for _ in self._fields]) - self.setQuery(query) - - -class SqlCompletionModel(QAbstractItemModel): - - """A sqlite-based model that provides data for the CompletionView. - - This model is a wrapper around one or more sql tables. The tables are all - stored in a single database in qutebrowser's cache directory. - - Top level indices represent categories, each of which is backed by a single - table. Child indices represent rows of those tables. + Top level indices represent categories. + Child indices represent rows of those tables. Attributes: column_widths: The width percentages of the columns used in the completion view. columns_to_filter: A list of indices of columns to apply the filter to. pattern: Current filter pattern, used for highlighting. - _categories: The category tables. + _categories: The sub-categories. """ def __init__(self, *, column_widths=(30, 70, 0), columns_to_filter=None, @@ -84,7 +48,6 @@ class SqlCompletionModel(QAbstractItemModel): self.columns_to_filter = columns_to_filter or [0] self.column_widths = column_widths self._categories = [] - self.srcmodel = self # TODO: dummy for compat with old API self.pattern = '' self.delete_cur_item = delete_cur_item @@ -94,7 +57,7 @@ class SqlCompletionModel(QAbstractItemModel): Args: idx: A QModelIndex Returns: - A _SqlCompletionCategory if the index points at one, else None + A category if the index points at one, else None """ # items hold an index to the parent category in their internalPointer # categories have an empty internalPointer @@ -102,7 +65,19 @@ class SqlCompletionModel(QAbstractItemModel): return self._categories[index.row()] return None - def new_category(self, name, *, select='*', where=None, sort_by=None, + def add_list(self, name, items): + """Add a list of items as a completion category. + + Args: + name: Title of the category. + items: List of tuples. + """ + cat = listcategory.ListCategory(name, items, parent=self) + filtermodel = sortfilter.CompletionFilterModel(cat, + self.columns_to_filter) + self._categories.append(filtermodel) + + def add_sqltable(self, name, *, select='*', where=None, sort_by=None, sort_order=None): """Create a new completion category and add it to this model. @@ -112,13 +87,11 @@ class SqlCompletionModel(QAbstractItemModel): where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Either 'asc' or 'desc', if sort_by is non-None - - Return: A new CompletionCategory. """ - cat = _SqlCompletionCategory(name, parent=self, sort_by=sort_by, - sort_order=sort_order, - select=select, where=where, - columns_to_filter=self.columns_to_filter) + cat = sqlcategory.SqlCategory(name, parent=self, sort_by=sort_by, + sort_order=sort_order, + select=select, where=where, + columns_to_filter=self.columns_to_filter) self._categories.append(cat) def data(self, index, role=Qt.DisplayRole): @@ -135,11 +108,11 @@ class SqlCompletionModel(QAbstractItemModel): return None if not index.parent().isValid(): if index.column() == 0: - return self._categories[index.row()].tablename + return self._categories[index.row()].name else: - table = self._categories[index.parent().row()] - idx = table.index(index.row(), index.column()) - return table.data(idx) + cat = self._categories[index.parent().row()] + idx = cat.index(index.row(), index.column()) + return cat.data(idx) def flags(self, index): """Return the item flags for index. @@ -171,7 +144,7 @@ class SqlCompletionModel(QAbstractItemModel): if parent.isValid(): if parent.column() != 0: return QModelIndex() - # store a pointer to the parent table in internalPointer + # store a pointer to the parent category in internalPointer return self.createIndex(row, col, self._categories[parent.row()]) return self.createIndex(row, col, None) @@ -183,11 +156,11 @@ class SqlCompletionModel(QAbstractItemModel): Args: index: The QModelIndex to get the parent index for. """ - parent_table = index.internalPointer() - if not parent_table: + parent_cat = index.internalPointer() + if not parent_cat: # categories have no parent return QModelIndex() - row = self._categories.index(parent_table) + row = self._categories.index(parent_cat) return self.createIndex(row, 0, None) def rowCount(self, parent=QModelIndex()): @@ -209,24 +182,24 @@ class SqlCompletionModel(QAbstractItemModel): return 3 def canFetchMore(self, parent): - """Override to forward the call to the tables.""" + """Override to forward the call to the categories.""" cat = self._cat_from_idx(parent) if cat: - return cat.canFetchMore() + return cat.canFetchMore(parent) return False def fetchMore(self, parent): - """Override to forward the call to the tables.""" + """Override to forward the call to the categories.""" cat = self._cat_from_idx(parent) if cat: - cat.fetchMore() + cat.fetchMore(parent) def count(self): """Return the count of non-category items.""" return sum(t.rowCount() for t in self._categories) def set_pattern(self, pattern): - """Set the filter pattern for all category tables. + """Set the filter pattern for all categories. This will apply to the fields indicated in columns_to_filter. @@ -236,19 +209,13 @@ class SqlCompletionModel(QAbstractItemModel): log.completion.debug("Setting completion pattern '{}'".format(pattern)) # TODO: should pattern be saved in the view layer instead? self.pattern = pattern - # escape to treat a user input % or _ as a literal, not a wildcard - pattern = pattern.replace('%', '\\%') - pattern = pattern.replace('_', '\\_') - # treat spaces as wildcards to match any of the typed words - pattern = re.sub(r' +', '%', pattern) - pattern = '%{}%'.format(pattern) for cat in self._categories: cat.set_pattern(pattern) def first_item(self): """Return the index of the first child (non-category) in the model.""" - for row, table in enumerate(self._categories): - if table.rowCount() > 0: + for row, cat in enumerate(self._categories): + if cat.rowCount() > 0: parent = self.index(row, 0) index = self.index(0, 0, parent) qtutils.ensure_valid(index) @@ -257,18 +224,11 @@ class SqlCompletionModel(QAbstractItemModel): def last_item(self): """Return the index of the last child (non-category) in the model.""" - for row, table in reversed(list(enumerate(self._categories))): - childcount = table.rowCount() + for row, cat in reversed(list(enumerate(self._categories))): + childcount = cat.rowCount() if childcount > 0: parent = self.index(row, 0) index = self.index(childcount - 1, 0, parent) qtutils.ensure_valid(index) return index return QModelIndex() - - -class SqlException(Exception): - - """Raised on an error interacting with the SQL database.""" - - pass diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index 7c7e3a0c6..3cb46d42d 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -20,17 +20,16 @@ """Functions that return config-related completion models.""" from qutebrowser.config import configdata, configexc -from qutebrowser.completion.models import base +from qutebrowser.completion.models import completionmodel from qutebrowser.utils import objreg def section(): """A CompletionModel filled with settings sections.""" - model = base.CompletionModel(column_widths=(20, 70, 10)) - cat = model.new_category("Sections") - for name in configdata.DATA: - desc = configdata.SECTION_DESC[name].splitlines()[0].strip() - model.new_item(cat, name, desc) + model = completionmodel.CompletionModel(column_widths=(20, 70, 10)) + sections = ((name, configdata.SECTION_DESC[name].splitlines()[0].strip()) + for name in configdata.DATA) + model.add_list("Sections", sections) return model @@ -40,12 +39,12 @@ def option(sectname): Args: sectname: The name of the config section this model shows. """ - model = base.CompletionModel(column_widths=(20, 70, 10)) - cat = model.new_category(sectname) + model = completionmodel.CompletionModel(column_widths=(20, 70, 10)) try: sectdata = configdata.DATA[sectname] except KeyError: return None + options = [] for name in sectdata: try: desc = sectdata.descriptions[name] @@ -57,7 +56,8 @@ def option(sectname): desc = desc.splitlines()[0] config = objreg.get('config') val = config.get(sectname, name, raw=True) - model.new_item(cat, name, desc, val) + options.append((name, desc, val)) + model.add_list(sectname, options) return model @@ -68,16 +68,16 @@ def value(sectname, optname): sectname: The name of the config section this model shows. optname: The name of the config option this model shows. """ - model = base.CompletionModel(column_widths=(20, 70, 10)) - cur_cat = model.new_category("Current/Default") + model = completionmodel.CompletionModel(column_widths=(20, 70, 10)) config = objreg.get('config') + try: - val = config.get(sectname, optname, raw=True) or '""' + current = config.get(sectname, optname, raw=True) or '""' except (configexc.NoSectionError, configexc.NoOptionError): return None - model.new_item(cur_cat, val, "Current value") - default_value = configdata.DATA[sectname][optname].default() or '""' - model.new_item(cur_cat, default_value, "Default value") + + default = configdata.DATA[sectname][optname].default() or '""' + if hasattr(configdata.DATA[sectname], 'valtype'): # Same type for all values (ValueList) vals = configdata.DATA[sectname].valtype.complete() @@ -87,8 +87,9 @@ def value(sectname, optname): "sections, but {} is not!".format(sectname)) # Different type for each value (KeyValue) vals = configdata.DATA[sectname][optname].typ.complete() + + model.add_list("Current/Default", [(current, "Current value"), + (default, "Default value")]) if vals is not None: - cat = model.new_category("Completions") - for (val, desc) in vals: - model.new_item(cat, val, desc) + model.add_list("Completions", vals) return model diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py new file mode 100644 index 000000000..7b222c3c0 --- /dev/null +++ b/qutebrowser/completion/models/listcategory.py @@ -0,0 +1,66 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""The base completion model for completion in the command line. + +Module attributes: + Role: An enum of user defined model roles. +""" + +from PyQt5.QtCore import Qt +from PyQt5.QtGui import QStandardItem, QStandardItemModel + + +class ListCategory(QStandardItemModel): + + """Expose a list of items as a category for the CompletionModel.""" + + def __init__(self, name, items, parent=None): + super().__init__(parent) + self.name = name + # self.setColumnCount(3) TODO needed? + # TODO: batch insert? + # TODO: can I just insert a tuple instead of a list? + for item in items: + self.appendRow([QStandardItem(x) for x in item]) + + def flags(self, index): + """Return the item flags for index. + + Override QAbstractItemModel::flags. + + Args: + index: The QModelIndex to get item flags for. + + Return: + The item flags, or Qt.NoItemFlags on error. + """ + if not index.isValid(): + return + + if index.parent().isValid(): + # item + return (Qt.ItemIsEnabled | Qt.ItemIsSelectable | + Qt.ItemNeverHasChildren) + else: + # category + return Qt.NoItemFlags + + def set_pattern(self, pattern): + pass diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index f5ed72f5d..d07ffaa88 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -22,30 +22,24 @@ from qutebrowser.config import config, configdata from qutebrowser.utils import objreg, log, qtutils from qutebrowser.commands import cmdutils -from qutebrowser.completion.models import base, sqlmodel +from qutebrowser.completion.models import completionmodel def command(): """A CompletionModel filled with non-hidden commands and descriptions.""" - model = base.CompletionModel(column_widths=(20, 60, 20)) + model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_aliases=True, include_hidden=False) - cat = model.new_category("Commands") - for (name, desc, misc) in cmdlist: - model.new_item(cat, name, desc, misc) + model.add_list("Commands", cmdlist) return model def helptopic(): """A CompletionModel filled with help topics.""" - model = base.CompletionModel() + model = completionmodel.CompletionModel() cmdlist = _get_cmd_completions(include_aliases=False, include_hidden=True, prefix=':') - cat = model.new_category("Commands") - for (name, desc, misc) in cmdlist: - model.new_item(cat, name, desc, misc) - - cat = model.new_category("Settings") + settings = [] for sectname, sectdata in configdata.DATA.items(): for optname in sectdata: try: @@ -57,34 +51,35 @@ def helptopic(): else: desc = desc.splitlines()[0] name = '{}->{}'.format(sectname, optname) - model.new_item(cat, name, desc) + settings.append((name, desc)) + + model.add_list("Commands", cmdlist) + model.add_list("Settings", settings) return model def quickmark(): """A CompletionModel filled with all quickmarks.""" - model = base.CompletionModel() - model = sqlmodel.SqlCompletionModel(column_widths=(30, 70, 0)) - model.new_category('Quickmarks') + model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) + model.add_sqltable('Quickmarks') return model def bookmark(): """A CompletionModel filled with all bookmarks.""" - model = base.CompletionModel() - model = sqlmodel.SqlCompletionModel(column_widths=(30, 70, 0)) - model.new_category('Bookmarks') + model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) + model.add_sqltable('Bookmarks') return model def session(): """A CompletionModel filled with session names.""" - model = base.CompletionModel() - cat = model.new_category("Sessions") + model = completionmodel.CompletionModel() try: - for name in objreg.get('session-manager').list_sessions(): - if not name.startswith('_'): - model.new_item(cat, name) + manager = objreg.get('session-manager') + sessions = ((name,) for name in manager.list_sessions() + if not name.startswith('_')) + model.add_list("Sessions", sessions) except OSError: log.completion.exception("Failed to list sessions!") return model @@ -111,7 +106,7 @@ def buffer(): window=int(win_id)) tabbed_browser.on_tab_close_requested(int(tab_index) - 1) - model = base.CompletionModel( + model = completionmodel.CompletionModel( column_widths=(6, 40, 54), delete_cur_item=delete_buffer, columns_to_filter=[idx_column, url_column, text_column]) @@ -121,12 +116,13 @@ def buffer(): window=win_id) if tabbed_browser.shutting_down: continue - c = model.new_category("{}".format(win_id)) + tabs = [] for idx in range(tabbed_browser.count()): tab = tabbed_browser.widget(idx) - model.new_item(c, "{}/{}".format(win_id, idx + 1), - tab.url().toDisplayString(), - tabbed_browser.page_title(idx)) + tabs.append(("{}/{}".format(win_id, idx + 1), + tab.url().toDisplayString(), + tabbed_browser.page_title(idx))) + model.add_list("{}".format(win_id), tabs) return model @@ -137,11 +133,9 @@ def bind(_key): _key: the key being bound. """ # TODO: offer a 'Current binding' completion based on the key. - model = base.CompletionModel(column_widths=(20, 60, 20)) + model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) - cat = model.new_category("Commands") - for (name, desc, misc) in cmdlist: - model.new_item(cat, name, desc, misc) + model.add_list("Commands", cmdlist) return model diff --git a/qutebrowser/completion/models/sortfilter.py b/qutebrowser/completion/models/sortfilter.py index 92d9aeaf1..fb79d002c 100644 --- a/qutebrowser/completion/models/sortfilter.py +++ b/qutebrowser/completion/models/sortfilter.py @@ -41,12 +41,14 @@ class CompletionFilterModel(QSortFilterProxyModel): a long time for some reason. """ - def __init__(self, source, parent=None): + def __init__(self, source, columns_to_filter, parent=None): super().__init__(parent) super().setSourceModel(source) self.srcmodel = source self.pattern = '' self.pattern_re = None + self.columns_to_filter = columns_to_filter + self.name = source.name def set_pattern(self, val): """Setter for pattern. @@ -64,41 +66,6 @@ class CompletionFilterModel(QSortFilterProxyModel): sortcol = 0 self.sort(sortcol) - def count(self): - """Get the count of non-toplevel items currently visible. - - Note this only iterates one level deep, as we only need root items - (categories) and children (items) in our model. - """ - count = 0 - for i in range(self.rowCount()): - cat = self.index(i, 0) - qtutils.ensure_valid(cat) - count += self.rowCount(cat) - return count - - def first_item(self): - """Return the first item in the model.""" - for i in range(self.rowCount()): - cat = self.index(i, 0) - qtutils.ensure_valid(cat) - if cat.model().hasChildren(cat): - index = self.index(0, 0, cat) - qtutils.ensure_valid(index) - return index - return QModelIndex() - - def last_item(self): - """Return the last item in the model.""" - for i in range(self.rowCount() - 1, -1, -1): - cat = self.index(i, 0) - qtutils.ensure_valid(cat) - if cat.model().hasChildren(cat): - index = self.index(self.rowCount(cat) - 1, 0, cat) - qtutils.ensure_valid(index) - return index - return QModelIndex() - def setSourceModel(self, model): """Override QSortFilterProxyModel's setSourceModel to clear pattern.""" log.completion.debug("Setting source model: {}".format(model)) @@ -119,10 +86,10 @@ class CompletionFilterModel(QSortFilterProxyModel): True if self.pattern is contained in item, or if it's a root item (category). False in all other cases """ - if parent == QModelIndex() or not self.pattern: + if not self.pattern: return True - for col in self.srcmodel.columns_to_filter: + for col in self.columns_to_filter: idx = self.srcmodel.index(row, col, parent) if not idx.isValid(): # pragma: no cover # this is a sanity check not hit by any test case diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py new file mode 100644 index 000000000..12e0534c1 --- /dev/null +++ b/qutebrowser/completion/models/sqlcategory.py @@ -0,0 +1,64 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""A completion model backed by SQL tables.""" + +import re + +from PyQt5.QtSql import QSqlQueryModel + +from qutebrowser.misc import sql + + +class SqlCategory(QSqlQueryModel): + """Wraps a SqlQuery for use as a completion category.""" + + def __init__(self, name, *, sort_by, sort_order, select, where, + columns_to_filter, parent=None): + super().__init__(parent=parent) + self.name = name + + query = sql.run_query('select * from {} limit 1'.format(name)) + self._fields = [query.record().fieldName(i) for i in columns_to_filter] + + querystr = 'select {} from {} where ('.format(select, name) + # the incoming pattern will have literal % and _ escaped with '\' + # we need to tell sql to treat '\' as an escape character + querystr += ' or '.join("{} like ? escape '\\'".format(f) + for f in self._fields) + querystr += ')' + if where: + querystr += ' and ' + where + + if sort_by: + assert sort_order == 'asc' or sort_order == 'desc' + querystr += ' order by {} {}'.format(sort_by, sort_order) + + self._querystr = querystr + self.set_pattern('') + + def set_pattern(self, pattern): + # escape to treat a user input % or _ as a literal, not a wildcard + pattern = pattern.replace('%', '\\%') + pattern = pattern.replace('_', '\\_') + # treat spaces as wildcards to match any of the typed words + pattern = re.sub(r' +', '%', pattern) + pattern = '%{}%'.format(pattern) + query = sql.run_query(self._querystr, [pattern for _ in self._fields]) + self.setQuery(query) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 0b357c3d7..a38e107d2 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -19,7 +19,7 @@ """Function to return the url completion model for the `open` command.""" -from qutebrowser.completion.models import sqlmodel +from qutebrowser.completion.models import completionmodel from qutebrowser.config import config from qutebrowser.utils import qtutils, log, objreg @@ -61,14 +61,16 @@ def url(): Used for the `open` command. """ - model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), - columns_to_filter=[_URLCOL, _TEXTCOL], - delete_cur_item=_delete_url) + model = completionmodel.CompletionModel( + column_widths=(40, 50, 10), + columns_to_filter=[_URLCOL, _TEXTCOL], + delete_cur_item=_delete_url) + timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) - model.new_category('Quickmarks', select='url, name') - model.new_category('Bookmarks') - model.new_category('History', + model.add_sqltable('Quickmarks', select='url, name') + model.add_sqltable('Bookmarks') + model.add_sqltable('History', sort_order='desc', sort_by='atime', select='url, title, {}'.format(select_time), where='not redirect') diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py new file mode 100644 index 000000000..4c1241d12 --- /dev/null +++ b/tests/unit/completion/test_completionmodel.py @@ -0,0 +1,146 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Tests for CompletionModel.""" + +import pytest + +from qutebrowser.completion.models import completionmodel, sortfilter + + +def _create_model(data, filter_cols=None): + """Create a completion model populated with the given data. + + data: A list of lists, where each sub-list represents a category, each + tuple in the sub-list represents an item, and each value in the + tuple represents the item data for that column + filter_cols: Columns to filter, or None for default. + """ + model = completionmodel.CompletionModel(columns_to_filter=filter_cols) + for catdata in data: + model.add_list('', catdata) + return model + + +def _extract_model_data(model): + """Express a model's data as a list for easier comparison. + + Return: A list of lists, where each sub-list represents a category, each + tuple in the sub-list represents an item, and each value in the + tuple represents the item data for that column + """ + data = [] + for i in range(0, model.rowCount()): + cat_idx = model.index(i, 0) + row = [] + for j in range(0, model.rowCount(cat_idx)): + row.append((model.data(cat_idx.child(j, 0)), + model.data(cat_idx.child(j, 1)), + model.data(cat_idx.child(j, 2)))) + data.append(row) + return data + + +@pytest.mark.parametrize('tree, first, last', [ + ([[('Aa',)]], 'Aa', 'Aa'), + ([[('Aa',)], [('Ba',)]], 'Aa', 'Ba'), + ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], + 'Aa', 'Ca'), + ([[], [('Ba',)]], 'Ba', 'Ba'), + ([[], [], [('Ca',)]], 'Ca', 'Ca'), + ([[], [], [('Ca',), ('Cb',)]], 'Ca', 'Cb'), + ([[('Aa',)], []], 'Aa', 'Aa'), + ([[('Aa',)], []], 'Aa', 'Aa'), + ([[('Aa',)], [], []], 'Aa', 'Aa'), + ([[('Aa',)], [], [('Ca',)]], 'Aa', 'Ca'), + ([[], []], None, None), +]) +def test_first_last_item(tree, first, last): + """Test that first() and last() return indexes to the first and last items. + + Args: + tree: Each list represents a completion category, with each string + being an item under that category. + first: text of the first item + last: text of the last item + """ + model = _create_model(tree) + assert model.data(model.first_item()) == first + assert model.data(model.last_item()) == last + + +@pytest.mark.parametrize('tree, expected', [ + ([[('Aa',)]], 1), + ([[('Aa',)], [('Ba',)]], 2), + ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], 6), + ([[], [('Ba',)]], 1), + ([[], [], [('Ca',)]], 1), + ([[], [], [('Ca',), ('Cb',)]], 2), + ([[('Aa',)], []], 1), + ([[('Aa',)], []], 1), + ([[('Aa',)], [], []], 1), + ([[('Aa',)], [], [('Ca',)]], 2), +]) +def test_count(tree, expected): + model = _create_model(tree) + assert model.count() == expected + + +@pytest.mark.parametrize('pattern, filter_cols, before, after', [ + ('foo', [0], + [[('foo', '', ''), ('bar', '', '')]], + [[('foo', '', '')]]), + + ('foo', [0], + [[('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')]], + [[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]]), + + ('foo', [0], + [[('foo', '', '')], [('bar', '', '')]], + [[('foo', '', '')], []]), + + # prefer foobar as it starts with the pattern + ('foo', [0], + [[('barfoo', '', ''), ('foobar', '', '')]], + [[('foobar', '', ''), ('barfoo', '', '')]]), + + # however, don't rearrange categories + ('foo', [0], + [[('barfoo', '', '')], [('foobar', '', '')]], + [[('barfoo', '', '')], [('foobar', '', '')]]), + + ('foo', [1], + [[('foo', 'bar', ''), ('bar', 'foo', '')]], + [[('bar', 'foo', '')]]), + + ('foo', [0, 1], + [[('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')]], + [[('foo', 'bar', ''), ('bar', 'foo', '')]]), + + ('foo', [0, 1, 2], + [[('foo', '', ''), ('bar', '')]], + [[('foo', '', '')]]), +]) +def test_set_pattern(pattern, filter_cols, before, after): + """Validate the filtering and sorting results of set_pattern.""" + # TODO: just test that it calls the mock on its child categories + model = _create_model(before, filter_cols) + model.set_pattern(pattern) + actual = _extract_model_data(model) + assert actual == after diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 2c8d96713..3db4dca5a 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -25,7 +25,7 @@ import pytest from PyQt5.QtGui import QStandardItem, QColor from qutebrowser.completion import completionwidget -from qutebrowser.completion.models import base, sortfilter +from qutebrowser.completion.models import completionmodel from qutebrowser.commands import cmdexc @@ -72,28 +72,25 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry, @pytest.fixture def simplemodel(completionview): - """A filter model wrapped around a completion model with one item.""" - model = base.CompletionModel() - cat = QStandardItem() - cat.appendRow(QStandardItem('foo')) - model.appendRow(cat) - return sortfilter.CompletionFilterModel(model, parent=completionview) + """A completion model with one item.""" + model = completionmodel.CompletionModel() + model.add_list('', [('foo'),]) + return model def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" - model = base.CompletionModel() - filtermodel = sortfilter.CompletionFilterModel(model) + model = completionmodel.CompletionModel() for i in range(3): - model.appendRow(QStandardItem(str(i))) - completionview.set_model(filtermodel) - assert completionview.model() is filtermodel + model.add_list(str(i), [('foo',)]) + completionview.set_model(model) + assert completionview.model() is model for i in range(model.rowCount()): - assert completionview.isExpanded(filtermodel.index(i, 0)) + assert completionview.isExpanded(model.index(i, 0)) def test_set_pattern(completionview): - model = sortfilter.CompletionFilterModel(base.CompletionModel()) + model = completionmodel.CompletionModel() model.set_pattern = unittest.mock.Mock() completionview.set_model(model, 'foo') model.set_pattern.assert_called_with('foo') @@ -159,15 +156,10 @@ def test_completion_item_focus(which, tree, expected, completionview, qtbot): successive movement. None implies no signal should be emitted. """ - model = base.CompletionModel() + model = completionmodel.CompletionModel() for catdata in tree: - cat = QStandardItem() - model.appendRow(cat) - for name in catdata: - cat.appendRow(QStandardItem(name)) - filtermodel = sortfilter.CompletionFilterModel(model, - parent=completionview) - completionview.set_model(filtermodel) + model.add_list('', (x,) for x in catdata) + completionview.set_model(model) for entry in expected: if entry is None: with qtbot.assertNotEmitted(completionview.selection_changed): @@ -187,10 +179,8 @@ def test_completion_item_focus_no_model(which, completionview, qtbot): """ with qtbot.assertNotEmitted(completionview.selection_changed): completionview.completion_item_focus(which) - model = base.CompletionModel() - filtermodel = sortfilter.CompletionFilterModel(model, - parent=completionview) - completionview.set_model(filtermodel) + model = completionmodel.CompletionModel() + completionview.set_model(model) completionview.set_model(None) with qtbot.assertNotEmitted(completionview.selection_changed): completionview.completion_item_focus(which) @@ -211,16 +201,12 @@ def test_completion_show(show, rows, quick_complete, completionview, config_stub.data['completion']['show'] = show config_stub.data['completion']['quick-complete'] = quick_complete - model = base.CompletionModel() + model = completionmodel.CompletionModel() for name in rows: - cat = QStandardItem() - model.appendRow(cat) - cat.appendRow(QStandardItem(name)) - filtermodel = sortfilter.CompletionFilterModel(model, - parent=completionview) + model.add_list('', [(name,)]) assert not completionview.isVisible() - completionview.set_model(filtermodel) + completionview.set_model(model) assert completionview.isVisible() == (show == 'always' and len(rows) > 0) completionview.completion_item_focus('next') expected = (show != 'never' and len(rows) > 0 and diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 224d7ed59..6d68cc8ab 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -228,12 +228,12 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): (':hide', '', ''), ], "Settings": [ - ('general->time', 'Is an illusion.', ''), - ('general->volume', 'Goes to 11', ''), - ('ui->gesture', 'Waggle your hands to control qutebrowser', ''), - ('ui->mind', 'Enable mind-control ui (experimental)', ''), - ('ui->voice', 'Whether to respond to voice commands', ''), - ('searchengines->DEFAULT', '', ''), + ('general->time', 'Is an illusion.', None), + ('general->volume', 'Goes to 11', None), + ('ui->gesture', 'Waggle your hands to control qutebrowser', None), + ('ui->mind', 'Enable mind-control ui (experimental)', None), + ('ui->voice', 'Whether to respond to voice commands', None), + ('searchengines->DEFAULT', '', None), ] }) @@ -342,7 +342,9 @@ def test_session_completion(qtmodeltester, session_manager_stub): qtmodeltester.check(model) _check_completions(model, { - "Sessions": [('default', '', ''), ('1', '', ''), ('2', '', '')] + "Sessions": [('default', None, None), + ('1', None, None), + ('2', None, None)] }) @@ -406,9 +408,9 @@ def test_setting_section_completion(qtmodeltester, monkeypatch, stubs): _check_completions(model, { "Sections": [ - ('general', 'General/miscellaneous options.', ''), - ('ui', 'General options related to the user interface.', ''), - ('searchengines', 'Definitions of search engines ...', ''), + ('general', 'General/miscellaneous options.', None), + ('ui', 'General options related to the user interface.', None), + ('searchengines', 'Definitions of search engines ...', None), ] }) @@ -462,12 +464,12 @@ def test_setting_value_completion(qtmodeltester, monkeypatch, stubs, _check_completions(model, { "Current/Default": [ - ('0', 'Current value', ''), - ('11', 'Default value', ''), + ('0', 'Current value', None), + ('11', 'Default value', None), ], "Completions": [ - ('0', '', ''), - ('11', '', ''), + ('0', '', None), + ('11', '', None), ] }) diff --git a/tests/unit/completion/test_sortfilter.py b/tests/unit/completion/test_sortfilter.py index 92d644810..7dadaa9dd 100644 --- a/tests/unit/completion/test_sortfilter.py +++ b/tests/unit/completion/test_sortfilter.py @@ -21,9 +21,10 @@ import pytest -from qutebrowser.completion.models import base, sortfilter +from qutebrowser.completion.models import listcategory, sortfilter +# TODO: merge listcategory and sortfilter def _create_model(data): """Create a completion model populated with the given data. @@ -31,11 +32,9 @@ def _create_model(data): tuple in the sub-list represents an item, and each value in the tuple represents the item data for that column """ - model = base.CompletionModel() + model = completionmodel.CompletionModel() for catdata in data: - cat = model.new_category('') - for itemdata in catdata: - model.new_item(cat, *itemdata) + cat = model.add_list(itemdata) return model @@ -72,7 +71,7 @@ def _extract_model_data(model): ('4', 'blah', False), ]) def test_filter_accepts_row(pattern, data, expected): - source_model = base.CompletionModel() + source_model = completionmodel.CompletionModel() cat = source_model.new_category('test') source_model.new_item(cat, data) @@ -86,35 +85,6 @@ def test_filter_accepts_row(pattern, data, expected): assert row_count == (1 if expected else 0) -@pytest.mark.parametrize('tree, first, last', [ - ([[('Aa',)]], 'Aa', 'Aa'), - ([[('Aa',)], [('Ba',)]], 'Aa', 'Ba'), - ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], - 'Aa', 'Ca'), - ([[], [('Ba',)]], 'Ba', 'Ba'), - ([[], [], [('Ca',)]], 'Ca', 'Ca'), - ([[], [], [('Ca',), ('Cb',)]], 'Ca', 'Cb'), - ([[('Aa',)], []], 'Aa', 'Aa'), - ([[('Aa',)], []], 'Aa', 'Aa'), - ([[('Aa',)], [], []], 'Aa', 'Aa'), - ([[('Aa',)], [], [('Ca',)]], 'Aa', 'Ca'), - ([[], []], None, None), -]) -def test_first_last_item(tree, first, last): - """Test that first() and last() return indexes to the first and last items. - - Args: - tree: Each list represents a completion category, with each string - being an item under that category. - first: text of the first item - last: text of the last item - """ - model = _create_model(tree) - filter_model = sortfilter.CompletionFilterModel(model) - assert filter_model.data(filter_model.first_item()) == first - assert filter_model.data(filter_model.last_item()) == last - - def test_set_source_model(): """Ensure setSourceModel sets source_model and clears the pattern.""" model1 = base.CompletionModel() @@ -131,24 +101,6 @@ def test_set_source_model(): assert not filter_model.pattern -@pytest.mark.parametrize('tree, expected', [ - ([[('Aa',)]], 1), - ([[('Aa',)], [('Ba',)]], 2), - ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], 6), - ([[], [('Ba',)]], 1), - ([[], [], [('Ca',)]], 1), - ([[], [], [('Ca',), ('Cb',)]], 2), - ([[('Aa',)], []], 1), - ([[('Aa',)], []], 1), - ([[('Aa',)], [], []], 1), - ([[('Aa',)], [], [('Ca',)]], 2), -]) -def test_count(tree, expected): - model = _create_model(tree) - filter_model = sortfilter.CompletionFilterModel(model) - assert filter_model.count() == expected - - @pytest.mark.parametrize('pattern, filter_cols, before, after', [ ('foo', [0], [[('foo', '', ''), ('bar', '', '')]], From 80619c88b39cdb16c86a096da2fbc0483185976b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 28 Feb 2017 08:43:11 -0500 Subject: [PATCH 048/161] Revert "Use SQL for quickmark/bookmark storage." This reverts commit fa1ebb03b70dfff4ac64038e67d9bab04b984de5. --- qutebrowser/browser/commands.py | 12 ++-- qutebrowser/browser/urlmarks.py | 110 ++++++++++++++++++++++---------- 2 files changed, 81 insertions(+), 41 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index e5ab0af21..8914d4ac2 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1260,15 +1260,13 @@ class CommandDispatcher: if name is None: url = self._current_url() try: - quickmark_manager.delete_by_qurl(url) + name = quickmark_manager.get_by_qurl(url) except urlmarks.DoesNotExistError as e: raise cmdexc.CommandError(str(e)) - else: - try: - quickmark_manager.delete(name) - except KeyError: - raise cmdexc.CommandError( - "Quickmark '{}' not found!".format(name)) + try: + quickmark_manager.delete(name) + except KeyError: + raise cmdexc.CommandError("Quickmark '{}' not found!".format(name)) @cmdutils.register(instance='command-dispatcher', scope='window') def bookmark_add(self, url=None, title=None, toggle=False): diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index 580d77881..013de408c 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -29,13 +29,14 @@ import os import html import os.path import functools +import collections -from PyQt5.QtCore import QUrl +from PyQt5.QtCore import pyqtSignal, QUrl, QObject from qutebrowser.utils import (message, usertypes, qtutils, urlutils, standarddir, objreg, log) from qutebrowser.commands import cmdutils -from qutebrowser.misc import lineparser, sql +from qutebrowser.misc import lineparser class Error(Exception): @@ -66,18 +67,29 @@ class AlreadyExistsError(Error): pass -class UrlMarkManager(sql.SqlTable): +class UrlMarkManager(QObject): """Base class for BookmarkManager and QuickmarkManager. Attributes: marks: An OrderedDict of all quickmarks/bookmarks. _lineparser: The LineParser used for the marks + + Signals: + changed: Emitted when anything changed. + added: Emitted when a new quickmark/bookmark was added. + removed: Emitted when an existing quickmark/bookmark was removed. """ - def __init__(self, name, fields, primary_key, parent=None): - """Initialize and read marks.""" - super().__init__(name, fields, primary_key, parent) + changed = pyqtSignal() + added = pyqtSignal(str, str) + removed = pyqtSignal(str) + + def __init__(self, parent=None): + """Initialize and read quickmarks.""" + super().__init__(parent) + + self.marks = collections.OrderedDict() self._init_lineparser() for line in self._lineparser: @@ -98,17 +110,31 @@ class UrlMarkManager(sql.SqlTable): def save(self): """Save the marks to disk.""" - self._lineparser.data = [' '.join(tpl) for tpl in self] + self._lineparser.data = [' '.join(tpl) for tpl in self.marks.items()] self._lineparser.save() + def delete(self, key): + """Delete a quickmark/bookmark. + + Args: + key: The key to delete (name for quickmarks, URL for bookmarks.) + """ + del self.marks[key] + self.changed.emit() + self.removed.emit(key) + class QuickmarkManager(UrlMarkManager): - """Manager for quickmarks.""" + """Manager for quickmarks. - def __init__(self, parent=None): - super().__init__('Quickmarks', ['name', 'url'], primary_key='name', - parent=parent) + The primary key for quickmarks is their *name*, this means: + + - self.marks maps names to URLs. + - changed gets emitted with the name as first argument and the URL as + second argument. + - removed gets emitted with the name as argument. + """ def _init_lineparser(self): self._lineparser = lineparser.LineParser( @@ -125,7 +151,7 @@ class QuickmarkManager(UrlMarkManager): except ValueError: message.error("Invalid quickmark '{}'".format(line)) else: - self.insert([key, url]) + self.marks[key] = url def prompt_save(self, url): """Prompt for a new quickmark name to be added and add it. @@ -165,32 +191,41 @@ class QuickmarkManager(UrlMarkManager): def set_mark(): """Really set the quickmark.""" - self.insert([name, url], replace=True) + self.marks[name] = url + self.changed.emit() + self.added.emit(name, url) log.misc.debug("Added quickmark {} for {}".format(name, url)) - if name in self: + if name in self.marks: message.confirm_async( title="Override existing quickmark?", yes_action=set_mark, default=True) else: set_mark() - def delete_by_qurl(self, url): - """Delete a quickmark by QUrl (as opposed to by name).""" + def get_by_qurl(self, url): + """Look up a quickmark by QUrl, returning its name. + + Takes O(n) time, where n is the number of quickmarks. + Use a name instead where possible. + """ qtutils.ensure_valid(url) urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) + try: - self.delete(urlstr, field='url') - except KeyError: - raise DoesNotExistError("Quickmark for '{}' not found!" - .format(urlstr)) + index = list(self.marks.values()).index(urlstr) + key = list(self.marks.keys())[index] + except ValueError: + raise DoesNotExistError( + "Quickmark for '{}' not found!".format(urlstr)) + return key def get(self, name): """Get the URL of the quickmark named name as a QUrl.""" - if name not in self: - raise DoesNotExistError("Quickmark '{}' does not exist!" - .format(name)) - urlstr = self[name].url + if name not in self.marks: + raise DoesNotExistError( + "Quickmark '{}' does not exist!".format(name)) + urlstr = self.marks[name] try: url = urlutils.fuzzy_url(urlstr, do_search=False) except urlutils.InvalidUrlError as e: @@ -201,11 +236,15 @@ class QuickmarkManager(UrlMarkManager): class BookmarkManager(UrlMarkManager): - """Manager for bookmarks.""" + """Manager for bookmarks. - def __init__(self, parent=None): - super().__init__('Bookmarks', ['url', 'title'], primary_key='url', - parent=parent) + The primary key for bookmarks is their *url*, this means: + + - self.marks maps URLs to titles. + - changed gets emitted with the URL as first argument and the title as + second argument. + - removed gets emitted with the URL as argument. + """ def _init_lineparser(self): bookmarks_directory = os.path.join(standarddir.config(), 'bookmarks') @@ -223,9 +262,10 @@ class BookmarkManager(UrlMarkManager): def _parse_line(self, line): parts = line.split(maxsplit=1) - urlstr = parts[0] - title = parts[1] if len(parts) == 2 else '' - self.insert([urlstr, title]) + if len(parts) == 2: + self.marks[parts[0]] = parts[1] + elif len(parts) == 1: + self.marks[parts[0]] = '' def add(self, url, title, *, toggle=False): """Add a new bookmark. @@ -246,12 +286,14 @@ class BookmarkManager(UrlMarkManager): urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) - if urlstr in self: + if urlstr in self.marks: if toggle: - self.delete(urlstr) + del self.marks[urlstr] return False else: raise AlreadyExistsError("Bookmark already exists!") else: - self.insert([urlstr, title]) + self.marks[urlstr] = title + self.changed.emit() + self.added.emit(title, urlstr) return True From 1d54688b0b3e74d59d18b9b4dc0f6509a96a4018 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 28 Feb 2017 08:44:17 -0500 Subject: [PATCH 049/161] Revert "Use SQL completer for quickmarks/bookmarks." This reverts commit bcf1520132df84552f69419f3b1cbf3ede20ccad. --- tests/unit/completion/test_models.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 6d68cc8ab..6e7e60958 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -238,6 +238,7 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): }) +@pytest.mark.skip def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() @@ -246,13 +247,14 @@ def test_quickmark_completion(qtmodeltester, quickmarks): _check_completions(model, { "Quickmarks": [ - ('aw', 'https://wiki.archlinux.org', None), - ('ddg', 'https://duckduckgo.com', None), - ('wiki', 'https://wikipedia.org', None), + ('aw', 'https://wiki.archlinux.org', ''), + ('ddg', 'https://duckduckgo.com', ''), + ('wiki', 'https://wikipedia.org', ''), ] }) +@pytest.mark.skip def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() @@ -261,9 +263,9 @@ def test_bookmark_completion(qtmodeltester, bookmarks): _check_completions(model, { "Bookmarks": [ - ('https://github.com', 'GitHub', None), - ('https://python.org', 'Welcome to Python.org', None), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), + ('https://github.com', 'GitHub', ''), + ('https://python.org', 'Welcome to Python.org', ''), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', ''), ] }) From ce3c5557123ee074db3a9092400f71f3beb38051 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 28 Feb 2017 08:50:55 -0500 Subject: [PATCH 050/161] Use list completion for bookmarks/quickmarks. The RFC on moving from plaintext to SQL storage (#2340) showed that many would be upset if bookmarks and quickmarks were no longer stored in plaintext. This commit uses list-based completion for quickmarks and bookmarks. Now the history storage can be moved from plaintext to an on-disk SQL database while leaving bookmarks and quickmarks as-is. --- qutebrowser/completion/models/miscmodels.py | 4 ++-- qutebrowser/completion/models/urlmodel.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index d07ffaa88..64a467a6b 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -61,14 +61,14 @@ def helptopic(): def quickmark(): """A CompletionModel filled with all quickmarks.""" model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) - model.add_sqltable('Quickmarks') + model.add_list('Quickmarks', objreg.get('quickmark-manager').marks.items()) return model def bookmark(): """A CompletionModel filled with all bookmarks.""" model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) - model.add_sqltable('Bookmarks') + model.add_list('Bookmarks', objreg.get('bookmark-manager').marks.items()) return model diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index a38e107d2..5b738c503 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -66,10 +66,13 @@ def url(): columns_to_filter=[_URLCOL, _TEXTCOL], delete_cur_item=_delete_url) + quickmarks = objreg.get('quickmark-manager').marks.items() + model.add_list('Quickmarks', ((url, name) for (name, url) in quickmarks)) + + model.add_list('Bookmarks', objreg.get('bookmark-manager').marks.items()) + timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) - model.add_sqltable('Quickmarks', select='url, name') - model.add_sqltable('Bookmarks') model.add_sqltable('History', sort_order='desc', sort_by='atime', select='url, title, {}'.format(select_time), From f95dff4d9e39abc3c29c514a3783ce632f96c369 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 3 Mar 2017 08:31:28 -0500 Subject: [PATCH 051/161] Decouple categories from completionmodel. Instead of add_list and add_sqltable, the completion model now supports add_category, and callees either pass in a SqlCategory or ListCategory. This makes unit testing much easier. This also folds CompletionFilterModel into the ListCategory class. --- qutebrowser/browser/commands.py | 9 +- qutebrowser/completion/completionwidget.py | 1 + .../completion/models/completionmodel.py | 32 +-- qutebrowser/completion/models/configmodel.py | 13 +- qutebrowser/completion/models/listcategory.py | 102 ++++++-- qutebrowser/completion/models/miscmodels.py | 21 +- qutebrowser/completion/models/sortfilter.py | 133 ---------- qutebrowser/completion/models/sqlcategory.py | 44 ++-- qutebrowser/completion/models/urlmodel.py | 22 +- scripts/dev/check_coverage.py | 2 - tests/helpers/fixtures.py | 18 ++ tests/unit/completion/test_completer.py | 3 +- tests/unit/completion/test_completionmodel.py | 163 ++++--------- .../unit/completion/test_completionwidget.py | 48 ++-- tests/unit/completion/test_listcategory.py | 70 ++++++ tests/unit/completion/test_models.py | 46 ++-- tests/unit/completion/test_sortfilter.py | 146 ----------- tests/unit/completion/test_sqlcateogry.py | 156 ++++++++++++ tests/unit/completion/test_sqlmodel.py | 230 ------------------ 19 files changed, 484 insertions(+), 775 deletions(-) delete mode 100644 qutebrowser/completion/models/sortfilter.py create mode 100644 tests/unit/completion/test_listcategory.py delete mode 100644 tests/unit/completion/test_sortfilter.py create mode 100644 tests/unit/completion/test_sqlcateogry.py delete mode 100644 tests/unit/completion/test_sqlmodel.py diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 8914d4ac2..3ec4c63c0 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -41,7 +41,7 @@ from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils, objreg, utils, typing) from qutebrowser.utils.usertypes import KeyMode from qutebrowser.misc import editor, guiprocess -from qutebrowser.completion.models import sortfilter, urlmodel, miscmodels +from qutebrowser.completion.models import urlmodel, miscmodels class CommandDispatcher: @@ -1024,10 +1024,9 @@ class CommandDispatcher: int(part) except ValueError: model = miscmodels.buffer() - sf = sortfilter.CompletionFilterModel(source=model) - sf.set_pattern(index) - if sf.count() > 0: - index = sf.data(sf.first_item()) + model.set_pattern(index) + if model.count() > 0: + index = model.data(model.first_item()) index_parts = index.split('/', 1) else: raise cmdexc.CommandError( diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index f6a980612..5e207d68d 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -275,6 +275,7 @@ class CompletionView(QTreeView): self.hide() return + model.setParent(self) old_model = self.model() if model is not old_model: sel_model = self.selectionModel() diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index b8dcc4601..84dd74b8f 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -24,7 +24,6 @@ import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from qutebrowser.utils import log, qtutils -from qutebrowser.completion.models import sortfilter, listcategory, sqlcategory class CompletionModel(QAbstractItemModel): @@ -65,33 +64,8 @@ class CompletionModel(QAbstractItemModel): return self._categories[index.row()] return None - def add_list(self, name, items): - """Add a list of items as a completion category. - - Args: - name: Title of the category. - items: List of tuples. - """ - cat = listcategory.ListCategory(name, items, parent=self) - filtermodel = sortfilter.CompletionFilterModel(cat, - self.columns_to_filter) - self._categories.append(filtermodel) - - def add_sqltable(self, name, *, select='*', where=None, sort_by=None, - sort_order=None): - """Create a new completion category and add it to this model. - - Args: - name: Name of category, and the table in the database. - select: A custom result column expression for the select statement. - where: An optional clause to filter out some rows. - sort_by: The name of the field to sort by, or None for no sorting. - sort_order: Either 'asc' or 'desc', if sort_by is non-None - """ - cat = sqlcategory.SqlCategory(name, parent=self, sort_by=sort_by, - sort_order=sort_order, - select=select, where=where, - columns_to_filter=self.columns_to_filter) + def add_category(self, cat): + """Add a completion category to the model.""" self._categories.append(cat) def data(self, index, role=Qt.DisplayRole): @@ -210,7 +184,7 @@ class CompletionModel(QAbstractItemModel): # TODO: should pattern be saved in the view layer instead? self.pattern = pattern for cat in self._categories: - cat.set_pattern(pattern) + cat.set_pattern(pattern, self.columns_to_filter) def first_item(self): """Return the index of the first child (non-category) in the model.""" diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index 3cb46d42d..663a0b7f7 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -20,7 +20,7 @@ """Functions that return config-related completion models.""" from qutebrowser.config import configdata, configexc -from qutebrowser.completion.models import completionmodel +from qutebrowser.completion.models import completionmodel, listcategory from qutebrowser.utils import objreg @@ -29,7 +29,7 @@ def section(): model = completionmodel.CompletionModel(column_widths=(20, 70, 10)) sections = ((name, configdata.SECTION_DESC[name].splitlines()[0].strip()) for name in configdata.DATA) - model.add_list("Sections", sections) + model.add_category(listcategory.ListCategory("Sections", sections)) return model @@ -57,7 +57,7 @@ def option(sectname): config = objreg.get('config') val = config.get(sectname, name, raw=True) options.append((name, desc, val)) - model.add_list(sectname, options) + model.add_category(listcategory.ListCategory(sectname, options)) return model @@ -88,8 +88,9 @@ def value(sectname, optname): # Different type for each value (KeyValue) vals = configdata.DATA[sectname][optname].typ.complete() - model.add_list("Current/Default", [(current, "Current value"), - (default, "Default value")]) + cur_cat = listcategory.ListCategory("Current/Default", + [(current, "Current value"), (default, "Default value")]) + model.add_category(cur_cat) if vals is not None: - model.add_list("Completions", vals) + model.add_category(listcategory.ListCategory("Completions", vals)) return model diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index 7b222c3c0..6251a7cac 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -23,44 +23,100 @@ Module attributes: Role: An enum of user defined model roles. """ -from PyQt5.QtCore import Qt +import re + +from PyQt5.QtCore import Qt, QSortFilterProxyModel, QModelIndex from PyQt5.QtGui import QStandardItem, QStandardItemModel +from qutebrowser.utils import qtutils, debug, log -class ListCategory(QStandardItemModel): + +class ListCategory(QSortFilterProxyModel): """Expose a list of items as a category for the CompletionModel.""" def __init__(self, name, items, parent=None): super().__init__(parent) self.name = name - # self.setColumnCount(3) TODO needed? - # TODO: batch insert? - # TODO: can I just insert a tuple instead of a list? + self.srcmodel = QStandardItemModel(parent=self) + self.pattern = '' + self.pattern_re = None for item in items: - self.appendRow([QStandardItem(x) for x in item]) + self.srcmodel.appendRow([QStandardItem(x) for x in item]) + self.setSourceModel(self.srcmodel) - def flags(self, index): - """Return the item flags for index. - - Override QAbstractItemModel::flags. + def set_pattern(self, val, columns_to_filter): + """Setter for pattern. Args: - index: The QModelIndex to get item flags for. + val: The value to set. + """ + with debug.log_time(log.completion, 'Setting filter pattern'): + self.columns_to_filter = columns_to_filter + self.pattern = val + val = re.sub(r' +', r' ', val) # See #1919 + val = re.escape(val) + val = val.replace(r'\ ', '.*') + self.pattern_re = re.compile(val, re.IGNORECASE) + self.invalidate() + sortcol = 0 + self.sort(sortcol) + + def filterAcceptsRow(self, row, parent): + """Custom filter implementation. + + Override QSortFilterProxyModel::filterAcceptsRow. + + Args: + row: The row of the item. + parent: The parent item QModelIndex. Return: - The item flags, or Qt.NoItemFlags on error. + True if self.pattern is contained in item, or if it's a root item + (category). False in all other cases """ - if not index.isValid(): - return + if not self.pattern: + return True - if index.parent().isValid(): - # item - return (Qt.ItemIsEnabled | Qt.ItemIsSelectable | - Qt.ItemNeverHasChildren) + for col in self.columns_to_filter: + idx = self.srcmodel.index(row, col, parent) + if not idx.isValid(): # pragma: no cover + # this is a sanity check not hit by any test case + continue + data = self.srcmodel.data(idx) + if not data: + continue + elif self.pattern_re.search(data): + return True + return False + + def lessThan(self, lindex, rindex): + """Custom sorting implementation. + + Prefers all items which start with self.pattern. Other than that, uses + normal Python string sorting. + + Args: + lindex: The QModelIndex of the left item (*left* < right) + rindex: The QModelIndex of the right item (left < *right*) + + Return: + True if left < right, else False + """ + qtutils.ensure_valid(lindex) + qtutils.ensure_valid(rindex) + + left = self.srcmodel.data(lindex) + right = self.srcmodel.data(rindex) + + leftstart = left.startswith(self.pattern) + rightstart = right.startswith(self.pattern) + + if leftstart and rightstart: + return left < right + elif leftstart: + return True + elif rightstart: + return False else: - # category - return Qt.NoItemFlags - - def set_pattern(self, pattern): - pass + return left < right diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 64a467a6b..368e1e6c6 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -22,14 +22,14 @@ from qutebrowser.config import config, configdata from qutebrowser.utils import objreg, log, qtutils from qutebrowser.commands import cmdutils -from qutebrowser.completion.models import completionmodel +from qutebrowser.completion.models import completionmodel, listcategory def command(): """A CompletionModel filled with non-hidden commands and descriptions.""" model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_aliases=True, include_hidden=False) - model.add_list("Commands", cmdlist) + model.add_category(listcategory.ListCategory("Commands", cmdlist)) return model @@ -53,22 +53,24 @@ def helptopic(): name = '{}->{}'.format(sectname, optname) settings.append((name, desc)) - model.add_list("Commands", cmdlist) - model.add_list("Settings", settings) + model.add_category(listcategory.ListCategory("Commands", cmdlist)) + model.add_category(listcategory.ListCategory("Settings", settings)) return model def quickmark(): """A CompletionModel filled with all quickmarks.""" model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) - model.add_list('Quickmarks', objreg.get('quickmark-manager').marks.items()) + marks = objreg.get('quickmark-manager').marks.items() + model.add_category(listcategory.ListCategory('Quickmarks', marks)) return model def bookmark(): """A CompletionModel filled with all bookmarks.""" model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) - model.add_list('Bookmarks', objreg.get('bookmark-manager').marks.items()) + marks = objreg.get('bookmark-manager').marks.items() + model.add_category(listcategory.ListCategory('Bookmarks', marks)) return model @@ -79,7 +81,7 @@ def session(): manager = objreg.get('session-manager') sessions = ((name,) for name in manager.list_sessions() if not name.startswith('_')) - model.add_list("Sessions", sessions) + model.add_category(listcategory.ListCategory("Sessions", sessions)) except OSError: log.completion.exception("Failed to list sessions!") return model @@ -122,7 +124,8 @@ def buffer(): tabs.append(("{}/{}".format(win_id, idx + 1), tab.url().toDisplayString(), tabbed_browser.page_title(idx))) - model.add_list("{}".format(win_id), tabs) + cat = listcategory.ListCategory("{}".format(win_id), tabs) + model.add_category(cat) return model @@ -135,7 +138,7 @@ def bind(_key): # TODO: offer a 'Current binding' completion based on the key. model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) - model.add_list("Commands", cmdlist) + model.add_category(listcategory.ListCategory("Commands", cmdlist)) return model diff --git a/qutebrowser/completion/models/sortfilter.py b/qutebrowser/completion/models/sortfilter.py deleted file mode 100644 index fb79d002c..000000000 --- a/qutebrowser/completion/models/sortfilter.py +++ /dev/null @@ -1,133 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2014-2017 Florian Bruhin (The Compiler) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""A filtering/sorting base model for completions. - -Contains: - CompletionFilterModel -- A QSortFilterProxyModel subclass for completions. -""" - -import re - -from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex - -from qutebrowser.utils import log, qtutils, debug - - -class CompletionFilterModel(QSortFilterProxyModel): - - """Subclass of QSortFilterProxyModel with custom filtering. - - Attributes: - pattern: The pattern to filter with. - srcmodel: The current source model. - Kept as attribute because calling `sourceModel` takes quite - a long time for some reason. - """ - - def __init__(self, source, columns_to_filter, parent=None): - super().__init__(parent) - super().setSourceModel(source) - self.srcmodel = source - self.pattern = '' - self.pattern_re = None - self.columns_to_filter = columns_to_filter - self.name = source.name - - def set_pattern(self, val): - """Setter for pattern. - - Args: - val: The value to set. - """ - with debug.log_time(log.completion, 'Setting filter pattern'): - self.pattern = val - val = re.sub(r' +', r' ', val) # See #1919 - val = re.escape(val) - val = val.replace(r'\ ', '.*') - self.pattern_re = re.compile(val, re.IGNORECASE) - self.invalidate() - sortcol = 0 - self.sort(sortcol) - - def setSourceModel(self, model): - """Override QSortFilterProxyModel's setSourceModel to clear pattern.""" - log.completion.debug("Setting source model: {}".format(model)) - self.set_pattern('') - super().setSourceModel(model) - self.srcmodel = model - - def filterAcceptsRow(self, row, parent): - """Custom filter implementation. - - Override QSortFilterProxyModel::filterAcceptsRow. - - Args: - row: The row of the item. - parent: The parent item QModelIndex. - - Return: - True if self.pattern is contained in item, or if it's a root item - (category). False in all other cases - """ - if not self.pattern: - return True - - for col in self.columns_to_filter: - idx = self.srcmodel.index(row, col, parent) - if not idx.isValid(): # pragma: no cover - # this is a sanity check not hit by any test case - continue - data = self.srcmodel.data(idx) - if not data: - continue - elif self.pattern_re.search(data): - return True - return False - - def lessThan(self, lindex, rindex): - """Custom sorting implementation. - - Prefers all items which start with self.pattern. Other than that, uses - normal Python string sorting. - - Args: - lindex: The QModelIndex of the left item (*left* < right) - rindex: The QModelIndex of the right item (left < *right*) - - Return: - True if left < right, else False - """ - qtutils.ensure_valid(lindex) - qtutils.ensure_valid(rindex) - - left = self.srcmodel.data(lindex) - right = self.srcmodel.data(rindex) - - leftstart = left.startswith(self.pattern) - rightstart = right.startswith(self.pattern) - - if leftstart and rightstart: - return left < right - elif leftstart: - return True - elif rightstart: - return False - else: - return left < right diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 12e0534c1..4f767abbf 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -29,36 +29,48 @@ from qutebrowser.misc import sql class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" - def __init__(self, name, *, sort_by, sort_order, select, where, - columns_to_filter, parent=None): + def __init__(self, name, *, sort_by=None, sort_order=None, select='*', + where=None, parent=None): + """Create a new completion category backed by a sql table. + + Args: + name: Name of category, and the table in the database. + select: A custom result column expression for the select statement. + where: An optional clause to filter out some rows. + sort_by: The name of the field to sort by, or None for no sorting. + sort_order: Either 'asc' or 'desc', if sort_by is non-None + """ super().__init__(parent=parent) self.name = name + self._sort_by = sort_by + self._sort_order = sort_order + self._select = select + self._where = where + self.set_pattern('', [0]) - query = sql.run_query('select * from {} limit 1'.format(name)) - self._fields = [query.record().fieldName(i) for i in columns_to_filter] + def set_pattern(self, pattern, columns_to_filter): + query = sql.run_query('select * from {} limit 1'.format(self.name)) + fields = [query.record().fieldName(i) for i in columns_to_filter] - querystr = 'select {} from {} where ('.format(select, name) + querystr = 'select {} from {} where ('.format(self._select, self.name) # the incoming pattern will have literal % and _ escaped with '\' # we need to tell sql to treat '\' as an escape character querystr += ' or '.join("{} like ? escape '\\'".format(f) - for f in self._fields) + for f in fields) querystr += ')' - if where: - querystr += ' and ' + where + if self._where: + querystr += ' and ' + self._where - if sort_by: - assert sort_order == 'asc' or sort_order == 'desc' - querystr += ' order by {} {}'.format(sort_by, sort_order) + if self._sort_by: + assert self._sort_order in ['asc', 'desc'] + querystr += ' order by {} {}'.format(self._sort_by, + self._sort_order) - self._querystr = querystr - self.set_pattern('') - - def set_pattern(self, pattern): # escape to treat a user input % or _ as a literal, not a wildcard pattern = pattern.replace('%', '\\%') pattern = pattern.replace('_', '\\_') # treat spaces as wildcards to match any of the typed words pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) - query = sql.run_query(self._querystr, [pattern for _ in self._fields]) + query = sql.run_query(querystr, [pattern for _ in fields]) self.setQuery(query) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 5b738c503..ea3f3f610 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -19,7 +19,8 @@ """Function to return the url completion model for the `open` command.""" -from qutebrowser.completion.models import completionmodel +from qutebrowser.completion.models import (completionmodel, listcategory, + sqlcategory) from qutebrowser.config import config from qutebrowser.utils import qtutils, log, objreg @@ -46,8 +47,7 @@ def _delete_url(completion): log.completion.debug('Deleting bookmark {}'.format(urlstr)) bookmark_manager = objreg.get('bookmark-manager') bookmark_manager.delete(urlstr) - else: - assert catname == 'Quickmarks', 'Unknown category {}'.format(catname) + elif catname == 'Quickmarks': quickmark_manager = objreg.get('quickmark-manager') sibling = index.sibling(index.row(), _TEXTCOL) qtutils.ensure_valid(sibling) @@ -66,15 +66,17 @@ def url(): columns_to_filter=[_URLCOL, _TEXTCOL], delete_cur_item=_delete_url) - quickmarks = objreg.get('quickmark-manager').marks.items() - model.add_list('Quickmarks', ((url, name) for (name, url) in quickmarks)) + quickmarks = ((url, name) for (name, url) + in objreg.get('quickmark-manager').marks.items()) + bookmarks = objreg.get('bookmark-manager').marks.items() - model.add_list('Bookmarks', objreg.get('bookmark-manager').marks.items()) + model.add_category(listcategory.ListCategory('Quickmarks', quickmarks)) + model.add_category(listcategory.ListCategory('Bookmarks', bookmarks)) timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) - model.add_sqltable('History', - sort_order='desc', sort_by='atime', - select='url, title, {}'.format(select_time), - where='not redirect') + hist_cat = sqlcategory.SqlCategory( + 'History', sort_order='desc', sort_by='atime', + select='url, title, {}'.format(select_time), where='not redirect') + model.add_category(hist_cat) return model diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 3b1611a1c..47e304a73 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -158,8 +158,6 @@ PERFECT_FILES = [ 'completion/models/base.py'), ('tests/unit/completion/test_models.py', 'completion/models/urlmodel.py'), - ('tests/unit/completion/test_sortfilter.py', - 'completion/models/sortfilter.py'), ] diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 0b69ad89f..87da45ce6 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -239,6 +239,24 @@ def host_blocker_stub(stubs): objreg.delete('host-blocker') +@pytest.fixture +def quickmark_manager_stub(stubs): + """Fixture which provides a fake quickmark manager object.""" + stub = stubs.QuickmarkManagerStub() + objreg.register('quickmark-manager', stub) + yield stub + objreg.delete('quickmark-manager') + + +@pytest.fixture +def bookmark_manager_stub(stubs): + """Fixture which provides a fake bookmark manager object.""" + stub = stubs.BookmarkManagerStub() + objreg.register('bookmark-manager', stub) + yield stub + objreg.delete('bookmark-manager') + + @pytest.fixture def session_manager_stub(stubs): """Fixture which provides a fake web-history object.""" diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index dce105c27..c407aeab8 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -193,11 +193,10 @@ def test_update_completion(txt, kind, pattern, pos_args, status_command_stub, completer_obj.schedule_completion_update() assert completion_widget_stub.set_model.call_count == 1 args = completion_widget_stub.set_model.call_args[0] - # the outer model is just for sorting; srcmodel is the completion model if kind is None: assert args[0] is None else: - model = args[0].srcmodel + model = args[0] assert model.kind == kind assert model.pos_args == pos_args assert args[1] == pattern diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 4c1241d12..7a61c0b84 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -19,128 +19,59 @@ """Tests for CompletionModel.""" +import sys import pytest +import hypothesis +from unittest import mock +from hypothesis import strategies -from qutebrowser.completion.models import completionmodel, sortfilter +from qutebrowser.completion.models import completionmodel -def _create_model(data, filter_cols=None): - """Create a completion model populated with the given data. - - data: A list of lists, where each sub-list represents a category, each - tuple in the sub-list represents an item, and each value in the - tuple represents the item data for that column - filter_cols: Columns to filter, or None for default. - """ - model = completionmodel.CompletionModel(columns_to_filter=filter_cols) - for catdata in data: - model.add_list('', catdata) - return model +@hypothesis.given(strategies.lists(min_size=0, max_size=3, + elements=strategies.integers(min_value=0, max_value=2**31))) +def test_first_last_item(counts): + """Test that first() and last() index to the first and last items.""" + model = completionmodel.CompletionModel() + for c in counts: + cat = mock.Mock() + cat.rowCount = mock.Mock(return_value=c) + model.add_category(cat) + nonempty = [i for i, rowCount in enumerate(counts) if rowCount > 0] + if not nonempty: + # with no items, first and last should be an invalid index + assert not model.first_item().isValid() + assert not model.last_item().isValid() + else: + first = nonempty[0] + last = nonempty[-1] + # first item of the first nonempty category + assert model.first_item().row() == 0 + assert model.first_item().parent().row() == first + # last item of the last nonempty category + assert model.last_item().row() == counts[last] - 1 + assert model.last_item().parent().row() == last -def _extract_model_data(model): - """Express a model's data as a list for easier comparison. - - Return: A list of lists, where each sub-list represents a category, each - tuple in the sub-list represents an item, and each value in the - tuple represents the item data for that column - """ - data = [] - for i in range(0, model.rowCount()): - cat_idx = model.index(i, 0) - row = [] - for j in range(0, model.rowCount(cat_idx)): - row.append((model.data(cat_idx.child(j, 0)), - model.data(cat_idx.child(j, 1)), - model.data(cat_idx.child(j, 2)))) - data.append(row) - return data +@hypothesis.given(strategies.lists(elements=strategies.integers(), + min_size=0, max_size=3)) +def test_count(counts): + model = completionmodel.CompletionModel() + for c in counts: + cat = mock.Mock(spec=['rowCount']) + cat.rowCount = mock.Mock(return_value=c) + model.add_category(cat) + assert model.count() == sum(counts) -@pytest.mark.parametrize('tree, first, last', [ - ([[('Aa',)]], 'Aa', 'Aa'), - ([[('Aa',)], [('Ba',)]], 'Aa', 'Ba'), - ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], - 'Aa', 'Ca'), - ([[], [('Ba',)]], 'Ba', 'Ba'), - ([[], [], [('Ca',)]], 'Ca', 'Ca'), - ([[], [], [('Ca',), ('Cb',)]], 'Ca', 'Cb'), - ([[('Aa',)], []], 'Aa', 'Aa'), - ([[('Aa',)], []], 'Aa', 'Aa'), - ([[('Aa',)], [], []], 'Aa', 'Aa'), - ([[('Aa',)], [], [('Ca',)]], 'Aa', 'Ca'), - ([[], []], None, None), -]) -def test_first_last_item(tree, first, last): - """Test that first() and last() return indexes to the first and last items. - - Args: - tree: Each list represents a completion category, with each string - being an item under that category. - first: text of the first item - last: text of the last item - """ - model = _create_model(tree) - assert model.data(model.first_item()) == first - assert model.data(model.last_item()) == last - - -@pytest.mark.parametrize('tree, expected', [ - ([[('Aa',)]], 1), - ([[('Aa',)], [('Ba',)]], 2), - ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], 6), - ([[], [('Ba',)]], 1), - ([[], [], [('Ca',)]], 1), - ([[], [], [('Ca',), ('Cb',)]], 2), - ([[('Aa',)], []], 1), - ([[('Aa',)], []], 1), - ([[('Aa',)], [], []], 1), - ([[('Aa',)], [], [('Ca',)]], 2), -]) -def test_count(tree, expected): - model = _create_model(tree) - assert model.count() == expected - - -@pytest.mark.parametrize('pattern, filter_cols, before, after', [ - ('foo', [0], - [[('foo', '', ''), ('bar', '', '')]], - [[('foo', '', '')]]), - - ('foo', [0], - [[('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')]], - [[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]]), - - ('foo', [0], - [[('foo', '', '')], [('bar', '', '')]], - [[('foo', '', '')], []]), - - # prefer foobar as it starts with the pattern - ('foo', [0], - [[('barfoo', '', ''), ('foobar', '', '')]], - [[('foobar', '', ''), ('barfoo', '', '')]]), - - # however, don't rearrange categories - ('foo', [0], - [[('barfoo', '', '')], [('foobar', '', '')]], - [[('barfoo', '', '')], [('foobar', '', '')]]), - - ('foo', [1], - [[('foo', 'bar', ''), ('bar', 'foo', '')]], - [[('bar', 'foo', '')]]), - - ('foo', [0, 1], - [[('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')]], - [[('foo', 'bar', ''), ('bar', 'foo', '')]]), - - ('foo', [0, 1, 2], - [[('foo', '', ''), ('bar', '')]], - [[('foo', '', '')]]), -]) -def test_set_pattern(pattern, filter_cols, before, after): +@hypothesis.given(strategies.text()) +def test_set_pattern(pat): """Validate the filtering and sorting results of set_pattern.""" - # TODO: just test that it calls the mock on its child categories - model = _create_model(before, filter_cols) - model.set_pattern(pattern) - actual = _extract_model_data(model) - assert actual == after + cols = [1, 2, 3] + model = completionmodel.CompletionModel(columns_to_filter=cols) + cats = [mock.Mock(spec=['set_pattern'])] * 3 + for c in cats: + c.set_pattern = mock.Mock() + model.add_category(c) + model.set_pattern(pat) + assert all(c.set_pattern.called_with([pat, cols]) for c in cats) diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 3db4dca5a..43722af42 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -19,13 +19,13 @@ """Tests for the CompletionView Object.""" -import unittest.mock +from unittest import mock import pytest from PyQt5.QtGui import QStandardItem, QColor from qutebrowser.completion import completionwidget -from qutebrowser.completion.models import completionmodel +from qutebrowser.completion.models import completionmodel, listcategory from qutebrowser.commands import cmdexc @@ -70,19 +70,11 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry, return view -@pytest.fixture -def simplemodel(completionview): - """A completion model with one item.""" - model = completionmodel.CompletionModel() - model.add_list('', [('foo'),]) - return model - - def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" model = completionmodel.CompletionModel() for i in range(3): - model.add_list(str(i), [('foo',)]) + cat = listcategory.ListCategory('', [('foo',)]) completionview.set_model(model) assert completionview.model() is model for i in range(model.rowCount()): @@ -91,7 +83,7 @@ def test_set_model(completionview): def test_set_pattern(completionview): model = completionmodel.CompletionModel() - model.set_pattern = unittest.mock.Mock() + model.set_pattern = mock.Mock() completionview.set_model(model, 'foo') model.set_pattern.assert_called_with('foo') @@ -158,7 +150,8 @@ def test_completion_item_focus(which, tree, expected, completionview, qtbot): """ model = completionmodel.CompletionModel() for catdata in tree: - model.add_list('', (x,) for x in catdata) + cat = listcategory.ListCategory('', ((x,) for x in catdata)) + model.add_category(cat) completionview.set_model(model) for entry in expected: if entry is None: @@ -203,7 +196,8 @@ def test_completion_show(show, rows, quick_complete, completionview, model = completionmodel.CompletionModel() for name in rows: - model.add_list('', [(name,)]) + cat = listcategory.ListCategory('', [(name,)]) + model.add_category(cat) assert not completionview.isVisible() completionview.set_model(model) @@ -217,27 +211,33 @@ def test_completion_show(show, rows, quick_complete, completionview, assert not completionview.isVisible() -def test_completion_item_del(completionview, simplemodel): +def test_completion_item_del(completionview): """Test that completion_item_del invokes delete_cur_item in the model.""" - simplemodel.srcmodel.delete_cur_item = unittest.mock.Mock() - completionview.set_model(simplemodel) + func = mock.Mock() + model = completionmodel.CompletionModel(delete_cur_item=func) + model.add_category(listcategory.ListCategory('', [('foo',)])) + completionview.set_model(model) completionview.completion_item_focus('next') completionview.completion_item_del() - assert simplemodel.srcmodel.delete_cur_item.called + assert func.called -def test_completion_item_del_no_selection(completionview, simplemodel): +def test_completion_item_del_no_selection(completionview): """Test that completion_item_del with no selected index.""" - simplemodel.srcmodel.delete_cur_item = unittest.mock.Mock() - completionview.set_model(simplemodel) + func = mock.Mock() + model = completionmodel.CompletionModel(delete_cur_item=func) + model.add_category(listcategory.ListCategory('', [('foo',)])) + completionview.set_model(model) with pytest.raises(cmdexc.CommandError): completionview.completion_item_del() - assert not simplemodel.srcmodel.delete_cur_item.called + assert not func.called -def test_completion_item_del_no_func(completionview, simplemodel): +def test_completion_item_del_no_func(completionview): """Test completion_item_del with no delete_cur_item in the model.""" - completionview.set_model(simplemodel) + model = completionmodel.CompletionModel() + model.add_category(listcategory.ListCategory('', [('foo',)])) + completionview.set_model(model) completionview.completion_item_focus('next') with pytest.raises(cmdexc.CommandError): completionview.completion_item_del() diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py new file mode 100644 index 000000000..0b4426bdb --- /dev/null +++ b/tests/unit/completion/test_listcategory.py @@ -0,0 +1,70 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2016 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Tests for CompletionFilterModel.""" + +import pytest + +from qutebrowser.completion.models import listcategory + + +def _validate(cat, expected): + """Check that a category contains the expected items in the given order. + + Args: + cat: The category to inspect. + expected: A list of tuples containing the expected items. + """ + assert cat.rowCount() == len(expected) + for row, items in enumerate(expected): + for col, item in enumerate(items): + assert cat.data(cat.index(row, col)) == item + + +@pytest.mark.parametrize('pattern, filter_cols, before, after', [ + ('foo', [0], + [('foo', '', ''), ('bar', '', '')], + [('foo', '', '')]), + + ('foo', [0], + [('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')], + [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]), + + # prefer foobar as it starts with the pattern + ('foo', [0], + [('barfoo', '', ''), ('foobar', '', '')], + [('foobar', '', ''), ('barfoo', '', '')]), + + ('foo', [1], + [('foo', 'bar', ''), ('bar', 'foo', '')], + [('bar', 'foo', '')]), + + ('foo', [0, 1], + [('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')], + [('foo', 'bar', ''), ('bar', 'foo', '')]), + + ('foo', [0, 1, 2], + [('foo', '', ''), ('bar', '')], + [('foo', '', '')]), +]) +def test_set_pattern(pattern, filter_cols, before, after): + """Validate the filtering and sorting results of set_pattern.""" + cat = listcategory.ListCategory('Foo', before) + cat.set_pattern(pattern, filter_cols) + _validate(cat, after) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 6e7e60958..48b3646bf 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -134,27 +134,25 @@ def _mock_view_index(model, category_num, child_num, qtbot): @pytest.fixture -def quickmarks(init_sql): - """Pre-populate the quickmark database.""" - table = sql.SqlTable('Quickmarks', ['name', 'url'], primary_key='name') - table.insert(['aw', 'https://wiki.archlinux.org']) - table.insert(['ddg', 'https://duckduckgo.com']) - table.insert(['wiki', 'https://wikipedia.org']) - objreg.register('quickmark-manager', table) - yield table - objreg.delete('quickmark-manager') +def quickmarks(quickmark_manager_stub): + """Pre-populate the quickmark-manager stub with some quickmarks.""" + quickmark_manager_stub.marks = collections.OrderedDict([ + ('aw', 'https://wiki.archlinux.org'), + ('ddg', 'https://duckduckgo.com'), + ('wiki', 'https://wikipedia.org'), + ]) + return quickmark_manager_stub @pytest.fixture -def bookmarks(init_sql): - """Pre-populate the bookmark database.""" - table = sql.SqlTable('Bookmarks', ['url', 'title'], primary_key='url') - table.insert(['https://github.com', 'GitHub']) - table.insert(['https://python.org', 'Welcome to Python.org']) - table.insert(['http://qutebrowser.org', 'qutebrowser | qutebrowser']) - objreg.register('bookmark-manager', table) - yield table - objreg.delete('bookmark-manager') +def bookmarks(bookmark_manager_stub): + """Pre-populate the bookmark-manager stub with some quickmarks.""" + bookmark_manager_stub.marks = collections.OrderedDict([ + ('https://github.com', 'GitHub'), + ('https://python.org', 'Welcome to Python.org'), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser'), + ]) + return bookmark_manager_stub @pytest.fixture @@ -315,9 +313,9 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, # delete item (1, 0) -> (bookmarks, 'https://github.com' ) view = _mock_view_index(model, 1, 0, qtbot) model.delete_cur_item(view) - assert 'https://github.com' not in bookmarks - assert 'https://python.org' in bookmarks - assert 'http://qutebrowser.org' in bookmarks + assert 'https://github.com' not in bookmarks.marks + assert 'https://python.org' in bookmarks.marks + assert 'http://qutebrowser.org' in bookmarks.marks def test_url_completion_delete_quickmark(qtmodeltester, config_stub, @@ -332,9 +330,9 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, # delete item (0, 1) -> (quickmarks, 'ddg' ) view = _mock_view_index(model, 0, 1, qtbot) model.delete_cur_item(view) - assert 'aw' in quickmarks - assert 'ddg' not in quickmarks - assert 'wiki' in quickmarks + assert 'aw' in quickmarks.marks + assert 'ddg' not in quickmarks.marks + assert 'wiki' in quickmarks.marks def test_session_completion(qtmodeltester, session_manager_stub): diff --git a/tests/unit/completion/test_sortfilter.py b/tests/unit/completion/test_sortfilter.py deleted file mode 100644 index 7dadaa9dd..000000000 --- a/tests/unit/completion/test_sortfilter.py +++ /dev/null @@ -1,146 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2015-2017 Florian Bruhin (The Compiler) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Tests for CompletionFilterModel.""" - -import pytest - -from qutebrowser.completion.models import listcategory, sortfilter - - -# TODO: merge listcategory and sortfilter -def _create_model(data): - """Create a completion model populated with the given data. - - data: A list of lists, where each sub-list represents a category, each - tuple in the sub-list represents an item, and each value in the - tuple represents the item data for that column - """ - model = completionmodel.CompletionModel() - for catdata in data: - cat = model.add_list(itemdata) - return model - - -def _extract_model_data(model): - """Express a model's data as a list for easier comparison. - - Return: A list of lists, where each sub-list represents a category, each - tuple in the sub-list represents an item, and each value in the - tuple represents the item data for that column - """ - data = [] - for i in range(0, model.rowCount()): - cat_idx = model.index(i, 0) - row = [] - for j in range(0, model.rowCount(cat_idx)): - row.append((model.data(cat_idx.child(j, 0)), - model.data(cat_idx.child(j, 1)), - model.data(cat_idx.child(j, 2)))) - data.append(row) - return data - - -@pytest.mark.parametrize('pattern, data, expected', [ - ('foo', 'barfoobar', True), - ('foo bar', 'barfoobar', True), - ('foo bar', 'barfoobar', True), - ('foo bar', 'barfoobazbar', True), - ('foo bar', 'barfoobazbar', True), - ('foo', 'barFOObar', True), - ('Foo', 'barfOObar', True), - ('ab', 'aonebtwo', False), - ('33', 'l33t', True), - ('x', 'blah', False), - ('4', 'blah', False), -]) -def test_filter_accepts_row(pattern, data, expected): - source_model = completionmodel.CompletionModel() - cat = source_model.new_category('test') - source_model.new_item(cat, data) - - filter_model = sortfilter.CompletionFilterModel(source_model) - filter_model.set_pattern(pattern) - assert filter_model.rowCount() == 1 # "test" category - idx = filter_model.index(0, 0) - assert idx.isValid() - - row_count = filter_model.rowCount(idx) - assert row_count == (1 if expected else 0) - - -def test_set_source_model(): - """Ensure setSourceModel sets source_model and clears the pattern.""" - model1 = base.CompletionModel() - model2 = base.CompletionModel() - filter_model = sortfilter.CompletionFilterModel(model1) - filter_model.set_pattern('foo') - # sourceModel() is cached as srcmodel, so make sure both match - assert filter_model.srcmodel is model1 - assert filter_model.sourceModel() is model1 - assert filter_model.pattern == 'foo' - filter_model.setSourceModel(model2) - assert filter_model.srcmodel is model2 - assert filter_model.sourceModel() is model2 - assert not filter_model.pattern - - -@pytest.mark.parametrize('pattern, filter_cols, before, after', [ - ('foo', [0], - [[('foo', '', ''), ('bar', '', '')]], - [[('foo', '', '')]]), - - ('foo', [0], - [[('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')]], - [[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]]), - - ('foo', [0], - [[('foo', '', '')], [('bar', '', '')]], - [[('foo', '', '')], []]), - - # prefer foobar as it starts with the pattern - ('foo', [0], - [[('barfoo', '', ''), ('foobar', '', '')]], - [[('foobar', '', ''), ('barfoo', '', '')]]), - - # however, don't rearrange categories - ('foo', [0], - [[('barfoo', '', '')], [('foobar', '', '')]], - [[('barfoo', '', '')], [('foobar', '', '')]]), - - ('foo', [1], - [[('foo', 'bar', ''), ('bar', 'foo', '')]], - [[('bar', 'foo', '')]]), - - ('foo', [0, 1], - [[('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')]], - [[('foo', 'bar', ''), ('bar', 'foo', '')]]), - - ('foo', [0, 1, 2], - [[('foo', '', ''), ('bar', '')]], - [[('foo', '', '')]]), -]) -def test_set_pattern(pattern, filter_cols, before, after): - """Validate the filtering and sorting results of set_pattern.""" - model = _create_model(before) - model.columns_to_filter = filter_cols - filter_model = sortfilter.CompletionFilterModel(model) - filter_model.set_pattern(pattern) - actual = _extract_model_data(filter_model) - assert actual == after diff --git a/tests/unit/completion/test_sqlcateogry.py b/tests/unit/completion/test_sqlcateogry.py new file mode 100644 index 000000000..1d25c5954 --- /dev/null +++ b/tests/unit/completion/test_sqlcateogry.py @@ -0,0 +1,156 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Test SQL-based completions.""" + +import pytest + +from qutebrowser.misc import sql +from qutebrowser.completion.models import sqlcategory + + +pytestmark = pytest.mark.usefixtures('init_sql') + + +def _validate(cat, expected): + """Check that a category contains the expected items in the given order. + + Args: + cat: The category to inspect. + expected: A list of tuples containing the expected items. + """ + assert cat.rowCount() == len(expected) + for row, items in enumerate(expected): + for col, item in enumerate(items): + assert cat.data(cat.index(row, col)) == item + + +@pytest.mark.parametrize('sort_by, sort_order, data, expected', [ + (None, 'asc', + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), + + ('a', 'asc', + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), + + ('a', 'desc', + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), + + ('b', 'asc', + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), + + ('b', 'desc', + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), + + ('c', 'asc', + [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], + [('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]), + + ('c', 'desc', + [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], + [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), +]) +def test_sorting(sort_by, sort_order, data, expected): + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + for row in data: + table.insert(row) + cat = sqlcategory.SqlCategory('Foo', sort_by=sort_by, + sort_order=sort_order) + _validate(cat, expected) + + +@pytest.mark.parametrize('pattern, filter_cols, before, after', [ + ('foo', [0], + [('foo', '', ''), ('bar', '', ''), ('aafobbb', '', '')], + [('foo',)]), + + ('foo', [0], + [('baz', 'bar', 'foo'), ('foo', '', ''), ('bar', 'foo', '')], + [('foo', '', '')]), + + ('foo', [0], + [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')], + [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]), + + ('foo', [1], + [('foo', 'bar', ''), ('bar', 'foo', '')], + [('bar', 'foo', '')]), + + ('foo', [0, 1], + [('foo', 'bar', ''), ('bar', 'foo', ''), ('biz', 'baz', 'foo')], + [('foo', 'bar', ''), ('bar', 'foo', '')]), + + ('foo', [0, 1, 2], + [('foo', '', ''), ('bar', '', ''), ('baz', 'bar', 'foo')], + [('foo', '', ''), ('baz', 'bar', 'foo')]), + + ('foo bar', [0], + [('foo', '', ''), ('bar foo', '', ''), ('xfooyybarz', '', '')], + [('xfooyybarz', '', '')]), + + ('foo%bar', [0], + [('foo%bar', '', ''), ('foo bar', '', ''), ('foobar', '', '')], + [('foo%bar', '', '')]), + + ('_', [0], + [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')], + [('a_b', '', ''), ('__a', '', '')]), + + ('%', [0, 1], + [('\\foo', '\\bar', '')], + []), + + ("can't", [0], + [("can't touch this", '', ''), ('a', '', '')], + [("can't touch this", '', '')]), +]) +def test_set_pattern(pattern, filter_cols, before, after): + """Validate the filtering and sorting results of set_pattern.""" + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + for row in before: + table.insert(row) + cat = sqlcategory.SqlCategory('Foo') + cat.set_pattern(pattern, filter_cols) + _validate(cat, after) + + +def test_select(): + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table.insert(['foo', 'bar', 'baz']) + cat = sqlcategory.SqlCategory('Foo', select='b, c, a') + _validate(cat, [('bar', 'baz', 'foo')]) + + +def test_where(): + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table.insert(['foo', 'bar', False]) + table.insert(['baz', 'biz', True]) + cat = sqlcategory.SqlCategory('Foo', where='not c') + _validate(cat, [('foo', 'bar', False)]) + + +def test_entry(): + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + assert hasattr(table.Entry, 'a') + assert hasattr(table.Entry, 'b') + assert hasattr(table.Entry, 'c') diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py deleted file mode 100644 index 8b9727dfd..000000000 --- a/tests/unit/completion/test_sqlmodel.py +++ /dev/null @@ -1,230 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Tests for the base sql completion model.""" - -import pytest - -from qutebrowser.misc import sql -from qutebrowser.completion.models import sqlmodel - - -pytestmark = pytest.mark.usefixtures('init_sql') - - -def _check_model(model, expected): - """Check that a model contains the expected items in the given order. - - Args: - expected: A list of form - [ - (cat, [(name, desc, misc), (name, desc, misc), ...]), - (cat, [(name, desc, misc), (name, desc, misc), ...]), - ... - ] - """ - assert model.rowCount() == len(expected) - for i, (expected_title, expected_items) in enumerate(expected): - catidx = model.index(i, 0) - assert model.data(catidx) == expected_title - assert model.rowCount(catidx) == len(expected_items) - for j, (name, desc, misc) in enumerate(expected_items): - assert model.data(model.index(j, 0, catidx)) == name - assert model.data(model.index(j, 1, catidx)) == desc - assert model.data(model.index(j, 2, catidx)) == misc - - -@pytest.mark.parametrize('rowcounts, expected', [ - ([0], 0), - ([1], 1), - ([2], 2), - ([0, 0], 0), - ([0, 0, 0], 0), - ([1, 1], 2), - ([3, 2, 1], 6), - ([0, 2, 0], 2), -]) -def test_count(rowcounts, expected): - model = sqlmodel.SqlCompletionModel() - for i, rowcount in enumerate(rowcounts): - name = 'Foo' + str(i) - table = sql.SqlTable(name, ['a'], primary_key='a') - for rownum in range(rowcount): - table.insert([rownum]) - model.new_category(name) - assert model.count() == expected - - -@pytest.mark.parametrize('sort_by, sort_order, data, expected', [ - (None, 'asc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), - - ('a', 'asc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - - ('a', 'desc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - - ('b', 'asc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - - ('b', 'desc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - - ('c', 'asc', - [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], - [('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]), - - ('c', 'desc', - [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], - [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), -]) -def test_sorting(sort_by, sort_order, data, expected): - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') - for row in data: - table.insert(row) - model = sqlmodel.SqlCompletionModel() - model.new_category('Foo', sort_by=sort_by, sort_order=sort_order) - _check_model(model, [('Foo', expected)]) - - -@pytest.mark.parametrize('pattern, filter_cols, before, after', [ - ('foo', [0], - [('A', [('foo', '', ''), ('bar', '', ''), ('aafobbb', '', '')])], - [('A', [('foo', '', '')])]), - - ('foo', [0], - [('A', [('baz', 'bar', 'foo'), ('foo', '', ''), ('bar', 'foo', '')])], - [('A', [('foo', '', '')])]), - - ('foo', [0], - [('A', [('foo', '', ''), ('bar', '', '')]), - ('B', [('foo', '', ''), ('bar', '', '')])], - [('A', [('foo', '', '')]), ('B', [('foo', '', '')])]), - - ('foo', [0], - [('A', [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')])], - [('A', [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')])]), - - ('foo', [0], - [('A', [('foo', '', '')]), ('B', [('bar', '', '')])], - [('A', [('foo', '', '')]), ('B', [])]), - - ('foo', [1], - [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])], - [('A', [('bar', 'foo', '')])]), - - ('foo', [0, 1], - [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])], - [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])]), - - ('foo', [0, 1, 2], - [('A', [('foo', '', ''), ('bar', '', '')])], - [('A', [('foo', '', '')])]), - - ('foo bar', [0], - [('A', [('foo', '', ''), ('bar foo', '', ''), ('xfooyybarz', '', '')])], - [('A', [('xfooyybarz', '', '')])]), - - ('foo%bar', [0], - [('A', [('foo%bar', '', ''), ('foo bar', '', ''), ('foobar', '', '')])], - [('A', [('foo%bar', '', '')])]), - - ('_', [0], - [('A', [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')])], - [('A', [('a_b', '', ''), ('__a', '', '')])]), - - ('%', [0, 1], - [('A', [('\\foo', '\\bar', '')])], - [('A', [])]), - - ("can't", [0], - [('A', [("can't touch this", '', ''), ('a', '', '')])], - [('A', [("can't touch this", '', '')])]), -]) -def test_set_pattern(pattern, filter_cols, before, after): - """Validate the filtering and sorting results of set_pattern.""" - model = sqlmodel.SqlCompletionModel(columns_to_filter=filter_cols) - for name, rows in before: - table = sql.SqlTable(name, ['a', 'b', 'c'], primary_key='a') - for row in rows: - table.insert(row) - model.new_category(name) - model.set_pattern(pattern) - _check_model(model, after) - - -@pytest.mark.parametrize('data, first, last', [ - ([('A', ['Aa'])], 'Aa', 'Aa'), - ([('A', ['Aa', 'Ba'])], 'Aa', 'Ba'), - ([('A', ['Aa', 'Ab', 'Ac']), ('B', ['Ba', 'Bb']), - ('C', ['Ca'])], 'Aa', 'Ca'), - ([('A', []), ('B', ['Ba'])], 'Ba', 'Ba'), - ([('A', []), ('B', []), ('C', ['Ca'])], 'Ca', 'Ca'), - ([('A', []), ('B', []), ('C', ['Ca', 'Cb'])], 'Ca', 'Cb'), - ([('A', ['Aa']), ('B', [])], 'Aa', 'Aa'), - ([('A', ['Aa']), ('B', []), ('C', [])], 'Aa', 'Aa'), - ([('A', ['Aa']), ('B', []), ('C', ['Ca'])], 'Aa', 'Ca'), - ([('A', []), ('B', [])], None, None), -]) -def test_first_last_item(data, first, last): - """Test that first() and last() return indexes to the first and last items. - - Args: - data: Input to _make_model - first: text of the first item - last: text of the last item - """ - model = sqlmodel.SqlCompletionModel() - for name, rows in data: - table = sql.SqlTable(name, ['a'], primary_key='a') - for row in rows: - table.insert([row]) - model.new_category(name) - assert model.data(model.first_item()) == first - assert model.data(model.last_item()) == last - - -def test_select(): - table = sql.SqlTable('test_select', ['a', 'b', 'c'], primary_key='a') - table.insert(['foo', 'bar', 'baz']) - model = sqlmodel.SqlCompletionModel() - model.new_category('test_select', select='b, c, a') - _check_model(model, [('test_select', [('bar', 'baz', 'foo')])]) - - -def test_where(): - table = sql.SqlTable('test_where', ['a', 'b', 'c'], primary_key='a') - table.insert(['foo', 'bar', False]) - table.insert(['baz', 'biz', True]) - model = sqlmodel.SqlCompletionModel() - model.new_category('test_where', where='not c') - _check_model(model, [('test_where', [('foo', 'bar', False)])]) - - -def test_entry(): - table = sql.SqlTable('test_entry', ['a', 'b', 'c'], primary_key='a') - assert hasattr(table.Entry, 'a') - assert hasattr(table.Entry, 'b') - assert hasattr(table.Entry, 'c') From 5dce6fa49415e083d19ef7fa63a725a54cc19ed3 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 21 Mar 2017 20:42:32 -0400 Subject: [PATCH 052/161] Fix pylint/flake8 errors --- qutebrowser/completion/models/completionmodel.py | 2 -- qutebrowser/completion/models/listcategory.py | 3 ++- tests/unit/completion/test_completionmodel.py | 2 -- tests/unit/completion/test_completionwidget.py | 6 +++--- tests/unit/completion/test_models.py | 1 - 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 84dd74b8f..0ee81fdf5 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -19,8 +19,6 @@ """A model that proxies access to one or more completion categories.""" -import re - from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from qutebrowser.utils import log, qtutils diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index 6251a7cac..d24fd74d0 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -25,7 +25,7 @@ Module attributes: import re -from PyQt5.QtCore import Qt, QSortFilterProxyModel, QModelIndex +from PyQt5.QtCore import QSortFilterProxyModel from PyQt5.QtGui import QStandardItem, QStandardItemModel from qutebrowser.utils import qtutils, debug, log @@ -41,6 +41,7 @@ class ListCategory(QSortFilterProxyModel): self.srcmodel = QStandardItemModel(parent=self) self.pattern = '' self.pattern_re = None + self.columns_to_filter = None for item in items: self.srcmodel.appendRow([QStandardItem(x) for x in item]) self.setSourceModel(self.srcmodel) diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 7a61c0b84..05af902a3 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -19,8 +19,6 @@ """Tests for CompletionModel.""" -import sys -import pytest import hypothesis from unittest import mock from hypothesis import strategies diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 43722af42..7985bfa5b 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -22,7 +22,7 @@ from unittest import mock import pytest -from PyQt5.QtGui import QStandardItem, QColor +from PyQt5.QtGui import QColor from qutebrowser.completion import completionwidget from qutebrowser.completion.models import completionmodel, listcategory @@ -74,10 +74,10 @@ def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" model = completionmodel.CompletionModel() for i in range(3): - cat = listcategory.ListCategory('', [('foo',)]) + model.add_category(listcategory.ListCategory('', [('foo',)])) completionview.set_model(model) assert completionview.model() is model - for i in range(model.rowCount()): + for i in range(3): assert completionview.isExpanded(model.index(i, 0)) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 48b3646bf..fa228847e 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -30,7 +30,6 @@ from qutebrowser.browser import history from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value from qutebrowser.misc import sql -from qutebrowser.utils import objreg def _check_completions(model, expected): From 93e0bfa410a429c680a8eba563d8ce3fa1d2b08e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 21 Mar 2017 21:41:13 -0400 Subject: [PATCH 053/161] Fix tests after sql completion rebase. --- qutebrowser/browser/qutescheme.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 23b009f6d..d12fb8f8e 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -212,7 +212,7 @@ def history_data(start_time): # noqa for item in history: # Skip redirects # Skip qute:// links - if item.redirect or item.url.scheme() == 'qute': + if item.redirect or item.url.startswith('qute://'): continue # Skip items out of time window @@ -241,11 +241,10 @@ def history_data(start_time): # noqa return # Use item's url as title if there's no title. - item_url = item.url.toDisplayString() - item_title = item.title if item.title else item_url + item_title = item.title if item.title else item.url item_time = int(item.atime * 1000) - yield {"url": item_url, "title": item_title, "time": item_time} + yield {"url": item.url, "title": item_title, "time": item_time} # if we reached here, we had reached the end of history yield {"next": int(last_item.atime if last_item else -1)} From 4296eed429289b463177e11ead28af9c294474bd Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 22 Mar 2017 19:55:03 -0400 Subject: [PATCH 054/161] Fix test_history cleanup failure. The test may be skipped if the PyQt5.QtWebKitWidget import fails, but the cleanup was still running and trying to delete a nonexistant web-history. --- tests/unit/browser/webkit/test_history.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 9d8711152..84af5fb46 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -325,12 +325,12 @@ def test_history_interface(qtbot, webview, hist_interface): @pytest.fixture def cleanup_init(): - yield # prevent test_init from leaking state - hist = objreg.get('web-history') - hist.setParent(None) - objreg.delete('web-history') + yield try: + hist = objreg.get('web-history') + hist.setParent(None) + objreg.delete('web-history') from PyQt5.QtWebKit import QWebHistoryInterface QWebHistoryInterface.setDefaultInterface(None) except: From 7d04f155c8a15b588a265af63d96445cd949c992 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 23 Mar 2017 08:29:22 -0400 Subject: [PATCH 055/161] Add missing docstring. --- qutebrowser/completion/models/sqlcategory.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 4f767abbf..693aaa841 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -49,6 +49,12 @@ class SqlCategory(QSqlQueryModel): self.set_pattern('', [0]) def set_pattern(self, pattern, columns_to_filter): + """Set the pattern used to filter results. + + Args: + pattern: string pattern to filter by. + columns_to_filter: indices of columns to apply pattern to. + """ query = sql.run_query('select * from {} limit 1'.format(self.name)) fields = [query.record().fieldName(i) for i in columns_to_filter] From 2eea115b3acd2c22d943d4c20850c244ff63bda0 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 23 Mar 2017 08:31:09 -0400 Subject: [PATCH 056/161] Rename sqlcategory and add to perfect_files. There was a typo in the file name. --- scripts/dev/check_coverage.py | 2 ++ .../completion/{test_sqlcateogry.py => test_sqlcategory.py} | 0 2 files changed, 2 insertions(+) rename tests/unit/completion/{test_sqlcateogry.py => test_sqlcategory.py} (100%) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 47e304a73..9904edd5b 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -158,6 +158,8 @@ PERFECT_FILES = [ 'completion/models/base.py'), ('tests/unit/completion/test_models.py', 'completion/models/urlmodel.py'), + ('tests/unit/completion/test_sqlcategory.py', + 'completion/models/sqlcategory.py'), ] diff --git a/tests/unit/completion/test_sqlcateogry.py b/tests/unit/completion/test_sqlcategory.py similarity index 100% rename from tests/unit/completion/test_sqlcateogry.py rename to tests/unit/completion/test_sqlcategory.py From b47c3b6a601b5ba1e8d9854c607867ca00523e78 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 23 Mar 2017 12:02:07 -0400 Subject: [PATCH 057/161] Test deleting a history entry from completion. Deleting a history entry should do nothing, but we want a test to ensure this and get 100% branch coverage for urlmodel. This also un-skips the bookmark/quickmark tests. --- tests/unit/completion/test_models.py | 30 ++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index fa228847e..9bc6fb289 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -167,6 +167,7 @@ def web_history(stubs, init_sql): datetime(2016, 3, 8).timestamp(), False]) table.insert(['https://github.com', 'https://github.com', datetime(2016, 5, 1).timestamp(), False]) + return table def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, @@ -235,7 +236,6 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): }) -@pytest.mark.skip def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() @@ -244,14 +244,13 @@ def test_quickmark_completion(qtmodeltester, quickmarks): _check_completions(model, { "Quickmarks": [ - ('aw', 'https://wiki.archlinux.org', ''), - ('ddg', 'https://duckduckgo.com', ''), - ('wiki', 'https://wikipedia.org', ''), + ('aw', 'https://wiki.archlinux.org', None), + ('ddg', 'https://duckduckgo.com', None), + ('wiki', 'https://wikipedia.org', None), ] }) -@pytest.mark.skip def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() @@ -260,9 +259,9 @@ def test_bookmark_completion(qtmodeltester, bookmarks): _check_completions(model, { "Bookmarks": [ - ('https://github.com', 'GitHub', ''), - ('https://python.org', 'Welcome to Python.org', ''), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', ''), + ('https://github.com', 'GitHub', None), + ('https://python.org', 'Welcome to Python.org', None), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ] }) @@ -334,6 +333,21 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, assert 'wiki' in quickmarks.marks +def test_url_completion_delete_history(qtmodeltester, config_stub, + web_history, quickmarks, bookmarks, + qtbot): + """Test that deleting a history entry is a noop.""" + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} + model = urlmodel.url() + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + hist_before = list(web_history) + view = _mock_view_index(model, 2, 0, qtbot) + model.delete_cur_item(view) + assert list(web_history) == hist_before + + def test_session_completion(qtmodeltester, session_manager_stub): session_manager_stub.sessions = ['default', '1', '2'] model = miscmodels.session() From de5be0dc5a1873704bd9971663bca2912eaf6f4e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 26 Mar 2017 16:52:10 -0400 Subject: [PATCH 058/161] Store history in an on-disk sqlite database. Instead of reading sqlite history from a file and storing it in an in-memory database, just directly use an on-disk database. This resolves #755, where history entries don't pop in to the completion menu immediately as they are still being read asynchronously for a few seconds after the browser starts. --- qutebrowser/app.py | 21 +-- qutebrowser/browser/history.py | 81 +--------- qutebrowser/misc/sql.py | 21 ++- scripts/dev/check_coverage.py | 2 + tests/helpers/fixtures.py | 5 +- tests/unit/browser/webkit/test_history.py | 176 +++++++--------------- tests/unit/misc/test_sql.py | 17 ++- 7 files changed, 92 insertions(+), 231 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 33726b058..0f98a9216 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -158,8 +158,6 @@ def init(args, crash_handler): QDesktopServices.setUrlHandler('https', open_desktopservices_url) QDesktopServices.setUrlHandler('qute', open_desktopservices_url) - QTimer.singleShot(10, functools.partial(_init_late_modules, args)) - log.init.debug("Init done!") crash_handler.raise_crashdlg() @@ -428,7 +426,7 @@ def _init_modules(args, crash_handler): keyconf.init(qApp) log.init.debug("Initializing sql...") - sql.init() + sql.init(os.path.join(standarddir.data(), 'history.sqlite')) log.init.debug("Initializing web history...") history.init(qApp) @@ -476,23 +474,6 @@ def _init_modules(args, crash_handler): browsertab.init() -def _init_late_modules(args): - """Initialize modules which can be inited after the window is shown.""" - log.init.debug("Reading web history...") - reader = objreg.get('web-history').async_read() - with debug.log_time(log.init, 'Reading history'): - while True: - QApplication.processEvents() - try: - next(reader) - except StopIteration: - break - except (OSError, UnicodeDecodeError) as e: - error.handle_fatal_exc(e, args, "Error while initializing!", - pre_text="Error while initializing") - sys.exit(usertypes.Exit.err_init) - - class Quitter: """Utility class to quit/restart the QApplication. diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index b6d328942..e7ff54037 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -26,7 +26,7 @@ from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl from qutebrowser.commands import cmdutils from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils, usertypes, message) -from qutebrowser.misc import lineparser, objects, sql +from qutebrowser.misc import objects, sql class Entry: @@ -118,14 +118,6 @@ class WebHistory(sql.SqlTable): self._saved_count tracks how many of those entries were already written to disk, so we can always append to the existing data. - Attributes: - _lineparser: The AppendLineParser used to save the history. - _new_history: A list of Entry items of the current session. - _saved_count: How many HistoryEntries have been written to disk. - _initial_read_started: Whether async_read was called. - _initial_read_done: Whether async_read has completed. - _temp_history: List of history entries from before async_read finished. - Signals: cleared: Emitted after the history is cleared. """ @@ -133,59 +125,13 @@ class WebHistory(sql.SqlTable): cleared = pyqtSignal() async_read_done = pyqtSignal() - def __init__(self, hist_dir, hist_name, parent=None): + def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], primary_key='url', parent=parent) - self._initial_read_started = False - self._initial_read_done = False - self._lineparser = lineparser.AppendLineParser(hist_dir, hist_name, - parent=self) - self._temp_history = [] - self._new_history = [] - self._saved_count = 0 def __repr__(self): return utils.get_repr(self, length=len(self)) - def async_read(self): - """Read the initial history.""" - if self._initial_read_started: - log.init.debug("Ignoring async_read() because reading is started.") - return - self._initial_read_started = True - - with self._lineparser.open(): - for line in self._lineparser: - yield - - line = line.rstrip() - if not line: - continue - - try: - entry = Entry.from_str(line) - except ValueError as e: - log.init.warning("Invalid history entry {!r}: {}!".format( - line, e)) - continue - - # This de-duplicates history entries; only the latest - # entry for each URL is kept. If you want to keep - # information about previous hits change the items in - # old_urls to be lists or change Entry to have a - # list of atimes. - self._add_entry(entry) - - self._initial_read_done = True - self.async_read_done.emit() - objreg.get('save-manager').add_saveable( - 'history', self.save, self.changed) - - for entry in self._temp_history: - self._add_entry(entry) - self._new_history.append(entry) - self._temp_history.clear() - def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, entry.atime, @@ -193,15 +139,7 @@ class WebHistory(sql.SqlTable): def get_recent(self): """Get the most recent history entries.""" - old = self._lineparser.get_recent() - return old + [str(e) for e in self._new_history] - - def save(self): - """Save the history to disk.""" - new = (str(e) for e in self._new_history[self._saved_count:]) - self._lineparser.new_data = new - self._lineparser.save() - self._saved_count = len(self._new_history) + return self.select(sort_by='atime', sort_order='desc', limit=100) @cmdutils.register(name='history-clear', instance='web-history') def clear(self, force=False): @@ -221,11 +159,7 @@ class WebHistory(sql.SqlTable): "history?") def _do_clear(self): - self._lineparser.clear() self.delete_all() - self._temp_history.clear() - self._new_history.clear() - self._saved_count = 0 self.cleared.emit() @pyqtSlot(QUrl, QUrl, str) @@ -263,11 +197,7 @@ class WebHistory(sql.SqlTable): if atime is None: atime = time.time() entry = Entry(atime, url, title, redirect=redirect) - if self._initial_read_done: - self._add_entry(entry) - self._new_history.append(entry) - else: - self._temp_history.append(entry) + self._add_entry(entry) def init(parent=None): @@ -276,8 +206,7 @@ def init(parent=None): Args: parent: The parent to use for WebHistory. """ - history = WebHistory(hist_dir=standarddir.data(), hist_name='history', - parent=parent) + history = WebHistory(parent=parent) objreg.register('web-history', history) if objects.backend == usertypes.Backend.QtWebKit: diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index c9249d362..eb1ad533c 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -34,14 +34,13 @@ class SqlException(Exception): pass -def init(): +def init(db_path): """Initialize the SQL database connection.""" database = QSqlDatabase.addDatabase('QSQLITE') - # In-memory database, see https://sqlite.org/inmemorydb.html - database.setDatabaseName(':memory:') + database.setDatabaseName(db_path) if not database.open(): - raise SqlException("Failed to open in-memory sqlite database: {}" - .format(database.lastError().text())) + raise SqlException("Failed to open sqlite database at {}: {}" + .format(db_path, database.lastError().text())) def close(): @@ -103,8 +102,8 @@ class SqlTable(QObject): super().__init__(parent) self._name = name self._primary_key = primary_key - run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))".format( - name, ','.join(fields), primary_key)) + run_query("CREATE TABLE IF NOT EXISTS {} ({}, PRIMARY KEY ({}))" + .format(name, ','.join(fields), primary_key)) # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) @@ -177,3 +176,11 @@ class SqlTable(QObject): """Remove all row from the table.""" run_query("DELETE FROM {}".format(self._name)) self.changed.emit() + + def select(self, sort_by, sort_order, limit): + """Remove all row from the table.""" + result = run_query('SELECT * FROM {} ORDER BY {} {} LIMIT {}' + .format(self._name, sort_by, sort_order, limit)) + while result.next(): + rec = result.record() + yield self.Entry(*[rec.value(i) for i in range(rec.count())]) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 9904edd5b..a2950b658 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -55,6 +55,8 @@ PERFECT_FILES = [ 'browser/history.py'), ('tests/unit/browser/webkit/test_history.py', 'browser/webkit/webkithistory.py'), + ('tests/unit/browser/webkit/test_history.py', + 'browser/history.py'), ('tests/unit/browser/webkit/http/test_http.py', 'browser/webkit/http.py'), ('tests/unit/browser/webkit/http/test_content_disposition.py', diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 87da45ce6..24087ed18 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -476,8 +476,9 @@ def short_tmpdir(): @pytest.fixture -def init_sql(): +def init_sql(data_tmpdir): """Initialize the SQL module, and shut it down after the test.""" - sql.init() + path = str(data_tmpdir / 'test.db') + sql.init(path) yield sql.close() diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 84af5fb46..9335cb239 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -38,47 +38,17 @@ def prerequisites(config_stub, fake_save_manager, init_sql): @pytest.fixture() def hist(tmpdir): - return history.WebHistory(hist_dir=str(tmpdir), hist_name='history') + return history.WebHistory() -def test_register_saveable(monkeypatch, qtbot, tmpdir, caplog, - fake_save_manager): - (tmpdir / 'filled-history').write('12345 http://example.com/ title') - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) - assert fake_save_manager.add_saveable.called - - -def test_async_read_twice(monkeypatch, qtbot, tmpdir, caplog): - (tmpdir / 'filled-history').write('\n'.join([ - '12345 http://example.com/ title', - '67890 http://example.com/', - '12345 http://qutebrowser.org/ blah', - ])) - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - next(hist.async_read()) - with pytest.raises(StopIteration): - next(hist.async_read()) - expected = "Ignoring async_read() because reading is started." - assert expected in (record.msg for record in caplog.records) - - -@pytest.mark.parametrize('redirect', [True, False]) -def test_adding_item_during_async_read(qtbot, hist, redirect): - """Check what happens when adding URL while reading the history.""" - url = 'http://www.example.com/' - hist.add_url(QUrl(url), redirect=redirect, atime=12345) - - with qtbot.waitSignal(hist.async_read_done): - list(hist.async_read()) - - assert not hist._temp_history - assert list(hist) == [(url, '', 12345, redirect)] +@pytest.fixture() +def mock_time(mocker): + m = mocker.patch('qutebrowser.browser.history.time') + m.time.return_value = 12345 + return 12345 def test_iter(hist): - list(hist.async_read()) - urlstr = 'http://www.example.com/' url = QUrl(urlstr) hist.add_url(url, atime=12345) @@ -88,7 +58,6 @@ def test_iter(hist): def test_len(hist): assert len(hist) == 0 - list(hist.async_read()) url = QUrl('http://www.example.com/') hist.add_url(url) @@ -96,101 +65,49 @@ def test_len(hist): assert len(hist) == 1 -@pytest.mark.parametrize('line', [ - '12345 http://example.com/ title', # with title - '67890 http://example.com/', # no title - '12345 http://qutebrowser.org/ ', # trailing space - ' ', - '', -]) -def test_read(tmpdir, line): - (tmpdir / 'filled-history').write(line + '\n') - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) +def test_updated_entries(tmpdir, hist): + hist.add_url(QUrl('http://example.com/'), atime=67890) + assert list(hist) == [('http://example.com/', '', 67890, False)] - -def test_updated_entries(tmpdir): - (tmpdir / 'filled-history').write('12345 http://example.com/\n' - '67890 http://example.com/\n') - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) - - assert hist['http://example.com/'] == ('http://example.com/', '', 67890, - False) hist.add_url(QUrl('http://example.com/'), atime=99999) - assert hist['http://example.com/'] == ('http://example.com/', '', 99999, - False) + assert list(hist) == [('http://example.com/', '', 99999, False)] -def test_invalid_read(tmpdir, caplog): - (tmpdir / 'filled-history').write('foobar\n12345 http://example.com/') - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - with caplog.at_level(logging.WARNING): - list(hist.async_read()) - - entries = list(hist) - - assert len(entries) == 1 - msg = "Invalid history entry 'foobar': 2 or 3 fields expected!" - assert msg in (rec.msg for rec in caplog.records) - - -def test_get_recent(tmpdir): - (tmpdir / 'filled-history').write('12345 http://example.com/') - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) - +def test_get_recent(hist): hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) - lines = hist.get_recent() - - expected = ['12345 http://example.com/', - '67890 http://www.qutebrowser.org/'] - assert lines == expected + hist.add_url(QUrl('http://example.com/'), atime=12345) + assert list(hist.get_recent()) == [ + ('http://www.qutebrowser.org/', '', 67890 , False), + ('http://example.com/', '', 12345, False), + ] -def test_save(tmpdir): - hist_file = tmpdir / 'filled-history' - hist_file.write('12345 http://example.com/\n') - - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) - +def test_save(tmpdir, hist): + hist.add_url(QUrl('http://example.com/'), atime=12345) hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) - hist.save() - lines = hist_file.read().splitlines() - expected = ['12345 http://example.com/', - '67890 http://www.qutebrowser.org/'] - assert lines == expected - - hist.add_url(QUrl('http://www.the-compiler.org/'), atime=99999) - hist.save() - expected.append('99999 http://www.the-compiler.org/') - - lines = hist_file.read().splitlines() - assert lines == expected + hist2 = history.WebHistory() + assert list(hist2) == [('http://example.com/', '', 12345, False), + ('http://www.qutebrowser.org/', '', 67890, False)] -def test_clear(qtbot, tmpdir): - hist_file = tmpdir / 'filled-history' - hist_file.write('12345 http://example.com/\n') +def test_clear(qtbot, tmpdir, hist, mocker): + hist.add_url(QUrl('http://example.com/')) + hist.add_url(QUrl('http://www.qutebrowser.org/')) - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) + m = mocker.patch('qutebrowser.browser.history.message.confirm_async') + hist.clear() + m.assert_called() + +def test_clear_force(qtbot, tmpdir, hist): + hist.add_url(QUrl('http://example.com/')) hist.add_url(QUrl('http://www.qutebrowser.org/')) with qtbot.waitSignal(hist.cleared): - hist._do_clear() + hist.clear(force=True) - assert not hist_file.read() - assert not hist._new_history - - hist.add_url(QUrl('http://www.the-compiler.org/'), atime=67890) - hist.save() - - lines = hist_file.read().splitlines() - assert lines == ['67890 http://www.the-compiler.org/'] + assert not len(hist) @pytest.mark.parametrize('item', [ @@ -198,21 +115,34 @@ def test_clear(qtbot, tmpdir): ('http://www.example.com', 12346, 'the title', True) ]) def test_add_item(qtbot, hist, item): - list(hist.async_read()) (url, atime, title, redirect) = item hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) assert hist[url] == (url, title, atime, redirect) -def test_add_item_redirect_update(qtbot, tmpdir, fake_save_manager): +def test_add_item_invalid(qtbot, hist, caplog): + with caplog.at_level(logging.WARNING): + hist.add_url(QUrl()) + assert not list(hist) + + +@pytest.mark.parametrize('level, url, req_url, expected', [ + (logging.DEBUG, 'a.com', 'a.com', [('a.com', 'title', 12345, False)]), + (logging.DEBUG, 'a.com', 'b.com', [('a.com', 'title', 12345, False), + ('b.com', 'title', 12345, True)]), + (logging.WARNING, 'a.com', '', [('a.com', 'title', 12345, False)]), + (logging.WARNING, '', '', []), +]) +def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog): + with caplog.at_level(level): + hist.add_from_tab(QUrl(url), QUrl(req_url), 'title') + assert set(list(hist)) == set(expected) + + +def test_add_item_redirect_update(qtbot, tmpdir, hist): """A redirect update added should override a non-redirect one.""" url = 'http://www.example.com/' - - hist_file = tmpdir / 'filled-history' - hist_file.write('12345 {}\n'.format(url)) - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) - + hist.add_url(QUrl(url), atime=5555) hist.add_url(QUrl(url), redirect=True, atime=67890) assert hist[url] == (url, '', 67890, True) diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index f4898b25d..4221bb022 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -28,9 +28,8 @@ pytestmark = pytest.mark.usefixtures('init_sql') def test_init(): sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - with pytest.raises(sql.SqlException): - # table name collision on 'Foo' - sql.SqlTable('Foo', ['foo', 'bar'], primary_key='foo') + # should not error if table already exists + sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') def test_insert(qtbot): @@ -54,6 +53,18 @@ def test_iter(): ('thirteen', 13, True)] +@pytest.mark.parametrize('rows, sort_by, sort_order, limit, result', [ + ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', 5, [(1, 6), (2, 5), (3, 4)]), + ([[2, 5], [1, 6], [3, 4]], 'a', 'desc', 3, [(3, 4), (2, 5), (1, 6)]), + ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]) +]) +def test_select(rows, sort_by, sort_order, limit, result): + table = sql.SqlTable('Foo', ['a', 'b'], primary_key='a') + for row in rows: + table.insert(row) + assert list(table.select(sort_by, sort_order, limit)) == result + + def test_replace(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') table.insert(['one', 1, False]) From 80647b062a6f28002dd283c8460a4ca7baed92fd Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 29 Mar 2017 08:59:16 -0400 Subject: [PATCH 059/161] Convert old history file to sqlite. If qutebrowser detects a history text file when it starts (~/.local/share/qutebrowser/history by default on Linux), it will import this file into the new sqlite database, then delete it. The read is done as a coroutine as it can take some time. --- qutebrowser/app.py | 23 +++++ qutebrowser/browser/history.py | 74 ++++++++------- qutebrowser/misc/sql.py | 52 +++++++++-- tests/unit/browser/webkit/test_history.py | 105 +++++----------------- 4 files changed, 135 insertions(+), 119 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 0f98a9216..e2e2d5107 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -158,6 +158,8 @@ def init(args, crash_handler): QDesktopServices.setUrlHandler('https', open_desktopservices_url) QDesktopServices.setUrlHandler('qute', open_desktopservices_url) + _import_history() + log.init.debug("Init done!") crash_handler.raise_crashdlg() @@ -474,6 +476,27 @@ def _init_modules(args, crash_handler): browsertab.init() +def _import_history(): + """Import a history text file into sqlite if it exists. + + In older versions of qutebrowser, history was stored in a text format. + This converts that file into the new sqlite format and removes it. + """ + path = os.path.join(standarddir.data(), 'history') + if not os.path.isfile(path): + return + + def action(): + with debug.log_time(log.init, 'Converting old history file to sqlite'): + objreg.get('web-history').read(path) + message.info('History import complete. Removing {}'.format(path)) + os.remove(path) + + # delay to give message time to appear before locking down for import + message.info('Converting {} to sqlite...'.format(path)) + QTimer.singleShot(100, action) + + class Quitter: """Utility class to quit/restart the QApplication. diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e7ff54037..8d10cdbbe 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -20,6 +20,7 @@ """Simple history which gets written to disk.""" import time +import os from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl @@ -70,37 +71,6 @@ class Entry: """Get the URL as a lossless string.""" return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) - @classmethod - def from_str(cls, line): - """Parse a history line like '12345 http://example.com title'.""" - data = line.split(maxsplit=2) - if len(data) == 2: - atime, url = data - title = "" - elif len(data) == 3: - atime, url, title = data - else: - raise ValueError("2 or 3 fields expected") - - url = QUrl(url) - if not url.isValid(): - raise ValueError("Invalid URL: {}".format(url.errorString())) - - # https://github.com/qutebrowser/qutebrowser/issues/670 - atime = atime.lstrip('\0') - - if '-' in atime: - atime, flags = atime.split('-') - else: - flags = '' - - if not set(flags).issubset('r'): - raise ValueError("Invalid flags {!r}".format(flags)) - - redirect = 'r' in flags - - return cls(atime, url, title, redirect=redirect) - class WebHistory(sql.SqlTable): @@ -199,6 +169,48 @@ class WebHistory(sql.SqlTable): entry = Entry(atime, url, title, redirect=redirect) self._add_entry(entry) + def _parse_entry(self, line): + """Parse a history line like '12345 http://example.com title'.""" + data = line.split(maxsplit=2) + if len(data) == 2: + atime, url = data + title = "" + elif len(data) == 3: + atime, url, title = data + else: + raise ValueError("2 or 3 fields expected") + + url = QUrl(url) + if not url.isValid(): + raise ValueError("Invalid URL: {}".format(url.errorString())) + + # https://github.com/qutebrowser/qutebrowser/issues/670 + atime = atime.lstrip('\0') + + if '-' in atime: + atime, flags = atime.split('-') + else: + flags = '' + + if not set(flags).issubset('r'): + raise ValueError("Invalid flags {!r}".format(flags)) + + redirect = 'r' in flags + + return (url, title, float(atime), bool(redirect)) + + def read(self, path): + """Import a text file into the sql database.""" + with open(path, 'r') as f: + rows = [] + for line in f: + try: + row = self._parse_entry(line.strip()) + rows.append(row) + except ValueError: + log.init.warning('Skipping history line {}'.format(line)) + self.insert_batch(rows) + def init(parent=None): """Initialize the web history. diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index eb1ad533c..d1d478302 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -55,16 +55,21 @@ def version(): return result.record().value(0) +def _prepare_query(querystr): + log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) + database = QSqlDatabase.database() + query = QSqlQuery(database) + query.prepare(querystr) + return query + + def run_query(querystr, values=None): """Run the given SQL query string on the database. Args: values: A list of positional parameter bindings. """ - log.sql.debug('Running SQL query: "{}"'.format(querystr)) - database = QSqlDatabase.database() - query = QSqlQuery(database) - query.prepare(querystr) + query = _prepare_query(querystr) for val in values or []: query.addBindValue(val) log.sql.debug('Query bindings: {}'.format(query.boundValues())) @@ -74,6 +79,28 @@ def run_query(querystr, values=None): return query +def run_batch(querystr, values): + """Run the given SQL query string on the database in batch mode. + + Args: + values: A list of lists, where each inner list contains positional + bindings for one run of the batch. + """ + query = _prepare_query(querystr) + transposed = [list(row) for row in zip(*values)] + for val in transposed: + query.addBindValue(val) + log.sql.debug('Batch Query bindings: {}'.format(query.boundValues())) + + db = QSqlDatabase.database() + db.transaction() + if not query.execBatch(): + raise SqlException('Failed to exec query "{}": "{}"'.format( + querystr, query.lastError().text())) + db.commit() + + return query + class SqlTable(QObject): """Interface to a sql table. @@ -102,8 +129,8 @@ class SqlTable(QObject): super().__init__(parent) self._name = name self._primary_key = primary_key - run_query("CREATE TABLE IF NOT EXISTS {} ({}, PRIMARY KEY ({}))" - .format(name, ','.join(fields), primary_key)) + run_query("CREATE TABLE IF NOT EXISTS {} ({})" + .format(name, ','.join(fields))) # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) @@ -172,6 +199,19 @@ class SqlTable(QObject): values) self.changed.emit() + def insert_batch(self, rows, replace=False): + """Performantly append multiple rows to the table. + + Args: + rows: A list of lists, where each sub-list is a row. + replace: If true, allow inserting over an existing primary key. + """ + cmd = "REPLACE" if replace else "INSERT" + paramstr = ','.join(['?'] * len(rows[0])) + run_batch("{} INTO {} values({})".format(cmd, self._name, paramstr), + rows) + self.changed.emit() + def delete_all(self): """Remove all row from the table.""" run_query("DELETE FROM {}".format(self._name)) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 9335cb239..e86950a82 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -148,88 +148,6 @@ def test_add_item_redirect_update(qtbot, tmpdir, hist): assert hist[url] == (url, '', 67890, True) -@pytest.mark.parametrize('line, expected', [ - ( - # old format without title - '12345 http://example.com/', - history.Entry(atime=12345, url=QUrl('http://example.com/'), title='',) - ), - ( - # trailing space without title - '12345 http://example.com/ ', - history.Entry(atime=12345, url=QUrl('http://example.com/'), title='',) - ), - ( - # new format with title - '12345 http://example.com/ this is a title', - history.Entry(atime=12345, url=QUrl('http://example.com/'), - title='this is a title') - ), - ( - # weird NUL bytes - '\x0012345 http://example.com/', - history.Entry(atime=12345, url=QUrl('http://example.com/'), title=''), - ), - ( - # redirect flag - '12345-r http://example.com/ this is a title', - history.Entry(atime=12345, url=QUrl('http://example.com/'), - title='this is a title', redirect=True) - ), -]) -def test_entry_parse_valid(line, expected): - entry = history.Entry.from_str(line) - assert entry == expected - - -@pytest.mark.parametrize('line', [ - '12345', # one field - '12345 ::', # invalid URL - 'xyz http://www.example.com/', # invalid timestamp - '12345-x http://www.example.com/', # invalid flags - '12345-r-r http://www.example.com/', # double flags -]) -def test_entry_parse_invalid(line): - with pytest.raises(ValueError): - history.Entry.from_str(line) - - -@hypothesis.given(strategies.text()) -def test_entry_parse_hypothesis(text): - """Make sure parsing works or gives us ValueError.""" - try: - history.Entry.from_str(text) - except ValueError: - pass - - -@pytest.mark.parametrize('entry, expected', [ - # simple - ( - history.Entry(12345, QUrl('http://example.com/'), "the title"), - "12345 http://example.com/ the title", - ), - # timestamp as float - ( - history.Entry(12345.678, QUrl('http://example.com/'), "the title"), - "12345 http://example.com/ the title", - ), - # no title - ( - history.Entry(12345.678, QUrl('http://example.com/'), ""), - "12345 http://example.com/", - ), - # redirect flag - ( - history.Entry(12345.678, QUrl('http://example.com/'), "", - redirect=True), - "12345-r http://example.com/", - ), -]) -def test_entry_str(entry, expected): - assert str(entry) == expected - - @pytest.fixture def hist_interface(): # pylint: disable=invalid-name @@ -298,3 +216,26 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): # For this to work, nothing can ever have called setDefaultInterface # before (so we need to test webengine before webkit) assert default_interface is None + + +def test_read(hist, tmpdir, caplog): + histfile = tmpdir / 'history' + histfile.write('''12345 http://example.com/ title + 12346 http://qutebrowser.org/ + 67890 http://example.com/path + + xyz http://example.com/bad-timestamp + 12345 + http://example.com/no-timestamp + 68891-r http://example.com/path/other + 68891-r-r http://example.com/double-flag''') + + with caplog.at_level(logging.WARNING): + hist.read(str(histfile)) + + assert list(hist) == [ + ('http://example.com/', 'title', 12345, False), + ('http://qutebrowser.org/', '', 12346, False), + ('http://example.com/path', '', 67890, False), + ('http://example.com/path/other', '', 68891, True) + ] From 8ff45331df32e35c214662056f8b5d72e789b8eb Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 4 Apr 2017 08:27:42 -0400 Subject: [PATCH 060/161] Clean up sql implementation. Now that sql is only used for history (not quickmarks/bookmarks) a number of functions are no longer needed. In addition, primary key support was removed as we actually need to support multiple entries for the same url with different access times. The completion model will have to handle this by selecting something like (url, title, max(atime)). This also fixes up a number of tests that were broken with the last few sql-related commits. --- qutebrowser/app.py | 2 +- qutebrowser/browser/history.py | 4 +- qutebrowser/misc/sql.py | 45 ++++-------------- qutebrowser/utils/version.py | 3 +- tests/end2end/features/test_history_bdd.py | 38 +++++++-------- tests/unit/browser/test_qutescheme.py | 3 +- tests/unit/browser/webkit/test_history.py | 19 +------- tests/unit/completion/test_models.py | 3 +- tests/unit/completion/test_sqlcategory.py | 10 ++-- tests/unit/misc/test_sql.py | 55 ++++------------------ 10 files changed, 49 insertions(+), 133 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index e2e2d5107..ccb474e69 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -481,7 +481,7 @@ def _import_history(): In older versions of qutebrowser, history was stored in a text format. This converts that file into the new sqlite format and removes it. - """ + """ path = os.path.join(standarddir.data(), 'history') if not os.path.isfile(path): return diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 8d10cdbbe..29e23750b 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -97,7 +97,7 @@ class WebHistory(sql.SqlTable): def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], - primary_key='url', parent=parent) + parent=parent) def __repr__(self): return utils.get_repr(self, length=len(self)) @@ -105,7 +105,7 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, entry.atime, - entry.redirect], replace=True) + entry.redirect]) def get_recent(self): """Get the most recent history entries.""" diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index d1d478302..707e0c4ce 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -101,6 +101,7 @@ def run_batch(querystr, values): return query + class SqlTable(QObject): """Interface to a sql table. @@ -108,7 +109,6 @@ class SqlTable(QObject): Attributes: Entry: The class wrapping row data from this table. _name: Name of the SQL table this wraps. - _primary_key: The primary key of the table. Signals: changed: Emitted when the table is modified. @@ -116,7 +116,7 @@ class SqlTable(QObject): changed = pyqtSignal() - def __init__(self, name, fields, primary_key, parent=None): + def __init__(self, name, fields, parent=None): """Create a new table in the sql database. Raises SqlException if the table already exists. @@ -124,11 +124,9 @@ class SqlTable(QObject): Args: name: Name of the table. fields: A list of field names. - primary_key: Name of the field to serve as the primary key. """ super().__init__(parent) self._name = name - self._primary_key = primary_key run_query("CREATE TABLE IF NOT EXISTS {} ({})" .format(name, ','.join(fields))) # pylint: disable=invalid-name @@ -141,74 +139,47 @@ class SqlTable(QObject): rec = result.record() yield self.Entry(*[rec.value(i) for i in range(rec.count())]) - def __contains__(self, key): - """Return whether the table contains the matching item. - - Args: - key: Primary key value to search for. - """ - query = run_query("SELECT * FROM {} where {} = ?".format( - self._name, self._primary_key), [key]) - return query.next() - def __len__(self): """Return the count of rows in the table.""" result = run_query("SELECT count(*) FROM {}".format(self._name)) result.next() return result.value(0) - def __getitem__(self, key): - """Retrieve the row matching the given key. - - Args: - key: Primary key value to fetch. - """ - result = run_query("SELECT * FROM {} where {} = ?".format( - self._name, self._primary_key), [key]) - result.next() - rec = result.record() - return self.Entry(*[rec.value(i) for i in range(rec.count())]) - - def delete(self, value, field=None): + def delete(self, value, field): """Remove all rows for which `field` equals `value`. Args: value: Key value to delete. - field: Field to use as the key, defaults to the primary key. + field: Field to use as the key. Return: The number of rows deleted. """ - field = field or self._primary_key query = run_query("DELETE FROM {} where {} = ?".format( self._name, field), [value]) if not query.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, values, replace=False): + def insert(self, values): """Append a row to the table. Args: values: A list of values to insert. - replace: If true, allow inserting over an existing primary key. """ - cmd = "REPLACE" if replace else "INSERT" paramstr = ','.join(['?'] * len(values)) - run_query("{} INTO {} values({})".format(cmd, self._name, paramstr), + run_query("INSERT INTO {} values({})".format(self._name, paramstr), values) self.changed.emit() - def insert_batch(self, rows, replace=False): + def insert_batch(self, rows): """Performantly append multiple rows to the table. Args: rows: A list of lists, where each sub-list is a row. - replace: If true, allow inserting over an existing primary key. """ - cmd = "REPLACE" if replace else "INSERT" paramstr = ','.join(['?'] * len(rows[0])) - run_batch("{} INTO {} values({})".format(cmd, self._name, paramstr), + run_batch("INSERT INTO {} values({})".format(self._name, paramstr), rows) self.changed.emit() diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index ac74f997a..faa890152 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -327,7 +327,8 @@ def version(): lines += ['pdf.js: {}'.format(_pdfjs_version())] - sql.init() + # we can use an in-memory database as we just want to query the version + sql.init('') lines += ['sqlite: {}'.format(sql.version())] sql.close() diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 1fee533eb..25995e9d0 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -20,31 +20,29 @@ import os.path import pytest_bdd as bdd + +from PyQt5.QtSql import QSqlDatabase, QSqlQuery + bdd.scenarios('history.feature') @bdd.then(bdd.parsers.parse("the history file should contain:\n{expected}")) def check_history(quteproc, httpbin, expected): - history_file = os.path.join(quteproc.basedir, 'data', 'history') - quteproc.send_cmd(':save history') - quteproc.wait_for(message=':save saved history') - - expected = expected.replace('(port)', str(httpbin.port)).splitlines() - - with open(history_file, 'r', encoding='utf-8') as f: - lines = [] - for line in f: - if not line.strip(): - continue - print('history line: ' + line) - atime, line = line.split(' ', maxsplit=1) - line = line.rstrip() - if '-' in atime: - flags = atime.split('-')[1] - line = '{} {}'.format(flags, line) - lines.append(line) - - assert lines == expected + path = os.path.join(quteproc.basedir, 'data', 'history.sqlite') + db = QSqlDatabase.addDatabase('QSQLITE') + db.setDatabaseName(path) + assert db.open(), 'Failed to open history database' + query = db.exec_('select * from History') + actual = [] + while query.next(): + rec = query.record() + url = rec.value(0) + title = rec.value(1) + redirect = rec.value(3) + actual.append('{} {} {}'.format('r' * redirect, url, title).strip()) + db = None + QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) + assert actual == expected.replace('(port)', str(httpbin.port)).splitlines() @bdd.then("the history file should be empty") diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index d5e60efab..86a322322 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -98,7 +98,7 @@ class TestHistoryHandler: @pytest.fixture def fake_web_history(self, fake_save_manager, tmpdir, init_sql): """Create a fake web-history and register it into objreg.""" - web_history = history.WebHistory(tmpdir.dirname, 'fake-history') + web_history = history.WebHistory() objreg.register('web-history', web_history) yield web_history objreg.delete('web-history') @@ -108,7 +108,6 @@ class TestHistoryHandler: """Create fake history.""" for item in entries: fake_web_history._add_entry(item) - fake_web_history.save() @pytest.mark.parametrize("start_time_offset, expected_item_count", [ (0, 4), diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index e86950a82..50c632f7d 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -65,14 +65,6 @@ def test_len(hist): assert len(hist) == 1 -def test_updated_entries(tmpdir, hist): - hist.add_url(QUrl('http://example.com/'), atime=67890) - assert list(hist) == [('http://example.com/', '', 67890, False)] - - hist.add_url(QUrl('http://example.com/'), atime=99999) - assert list(hist) == [('http://example.com/', '', 99999, False)] - - def test_get_recent(hist): hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) hist.add_url(QUrl('http://example.com/'), atime=12345) @@ -117,7 +109,7 @@ def test_clear_force(qtbot, tmpdir, hist): def test_add_item(qtbot, hist, item): (url, atime, title, redirect) = item hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) - assert hist[url] == (url, title, atime, redirect) + assert list(hist) == [(url, title, atime, redirect)] def test_add_item_invalid(qtbot, hist, caplog): @@ -139,15 +131,6 @@ def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog): assert set(list(hist)) == set(expected) -def test_add_item_redirect_update(qtbot, tmpdir, hist): - """A redirect update added should override a non-redirect one.""" - url = 'http://www.example.com/' - hist.add_url(QUrl(url), atime=5555) - hist.add_url(QUrl(url), redirect=True, atime=67890) - - assert hist[url] == (url, '', 67890, True) - - @pytest.fixture def hist_interface(): # pylint: disable=invalid-name diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 9bc6fb289..e071f1584 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -157,8 +157,7 @@ def bookmarks(bookmark_manager_stub): @pytest.fixture def web_history(stubs, init_sql): """Pre-populate the web-history database.""" - table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect'], - primary_key='url') + table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect']) table.insert(['http://some-redirect.example.com', 'redirect', datetime(2016, 9, 5).timestamp(), True]) table.insert(['http://qutebrowser.org', 'qutebrowser', diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 1d25c5954..03288f016 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -71,7 +71,7 @@ def _validate(cat, expected): [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), ]) def test_sorting(sort_by, sort_order, data, expected): - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: table.insert(row) cat = sqlcategory.SqlCategory('Foo', sort_by=sort_by, @@ -126,7 +126,7 @@ def test_sorting(sort_by, sort_order, data, expected): ]) def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: table.insert(row) cat = sqlcategory.SqlCategory('Foo') @@ -135,14 +135,14 @@ def test_set_pattern(pattern, filter_cols, before, after): def test_select(): - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert(['foo', 'bar', 'baz']) cat = sqlcategory.SqlCategory('Foo', select='b, c, a') _validate(cat, [('bar', 'baz', 'foo')]) def test_where(): - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert(['foo', 'bar', False]) table.insert(['baz', 'biz', True]) cat = sqlcategory.SqlCategory('Foo', where='not c') @@ -150,7 +150,7 @@ def test_where(): def test_entry(): - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b', 'c']) assert hasattr(table.Entry, 'a') assert hasattr(table.Entry, 'b') assert hasattr(table.Entry, 'c') diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 4221bb022..387b5ce46 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -27,24 +27,21 @@ pytestmark = pytest.mark.usefixtures('init_sql') def test_init(): - sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + sql.SqlTable('Foo', ['name', 'val', 'lucky']) # should not error if table already exists - sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + sql.SqlTable('Foo', ['name', 'val', 'lucky']) def test_insert(qtbot): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) with qtbot.waitSignal(table.changed): table.insert(['one', 1, False]) with qtbot.waitSignal(table.changed): table.insert(['wan', 1, False]) - with pytest.raises(sql.SqlException): - # duplicate primary key - table.insert(['one', 1, False]) def test_iter(): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) table.insert(['one', 1, False]) table.insert(['nine', 9, False]) table.insert(['thirteen', 13, True]) @@ -59,29 +56,21 @@ def test_iter(): ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]) ]) def test_select(rows, sort_by, sort_order, limit, result): - table = sql.SqlTable('Foo', ['a', 'b'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b']) for row in rows: table.insert(row) assert list(table.select(sort_by, sort_order, limit)) == result -def test_replace(qtbot): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert(['one', 1, False]) - with qtbot.waitSignal(table.changed): - table.insert(['one', 1, True], replace=True) - assert list(table) == [('one', 1, True)] - - def test_delete(qtbot): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) table.insert(['one', 1, False]) table.insert(['nine', 9, False]) table.insert(['thirteen', 13, True]) with pytest.raises(KeyError): - table.delete('nope') + table.delete('nope', 'name') with qtbot.waitSignal(table.changed): - table.delete('thirteen') + table.delete('thirteen', 'name') assert list(table) == [('one', 1, False), ('nine', 9, False)] with qtbot.waitSignal(table.changed): table.delete(False, field='lucky') @@ -89,7 +78,7 @@ def test_delete(qtbot): def test_len(): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) assert len(table) == 0 table.insert(['one', 1, False]) assert len(table) == 1 @@ -99,32 +88,8 @@ def test_len(): assert len(table) == 3 -def test_contains(): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) - assert 'oone' not in table - assert 'ninee' not in table - assert 1 not in table - assert '*' not in table - assert 'one' in table - assert 'nine' in table - assert 'thirteen' in table - - -def test_getitem(): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) - assert table['one'] == ('one', 1, False) - assert table['nine'] == ('nine', 9, False) - assert table['thirteen'] == ('thirteen', 13, True) - - def test_delete_all(qtbot): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) table.insert(['one', 1, False]) table.insert(['nine', 9, False]) table.insert(['thirteen', 13, True]) From 3e63b62d6e43717b1b0c900125bf0a7ba55c9a97 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 6 Apr 2017 12:02:43 -0400 Subject: [PATCH 061/161] Fix pylint/flake8 for sql work. --- qutebrowser/app.py | 4 ++-- qutebrowser/browser/history.py | 6 ++---- tests/end2end/features/test_history_bdd.py | 2 +- tests/unit/browser/webkit/test_history.py | 4 +--- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index ccb474e69..dd73827af 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -479,8 +479,8 @@ def _init_modules(args, crash_handler): def _import_history(): """Import a history text file into sqlite if it exists. - In older versions of qutebrowser, history was stored in a text format. - This converts that file into the new sqlite format and removes it. + In older versions of qutebrowser, history was stored in a text format. + This converts that file into the new sqlite format and removes it. """ path = os.path.join(standarddir.data(), 'history') if not os.path.isfile(path): diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 29e23750b..1a5614fd6 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -20,13 +20,11 @@ """Simple history which gets written to disk.""" import time -import os from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl from qutebrowser.commands import cmdutils -from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils, - usertypes, message) +from qutebrowser.utils import utils, objreg, log, qtutils, usertypes, message from qutebrowser.misc import objects, sql @@ -201,7 +199,7 @@ class WebHistory(sql.SqlTable): def read(self, path): """Import a text file into the sql database.""" - with open(path, 'r') as f: + with open(path, 'r', encoding='utf-8') as f: rows = [] for line in f: try: diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 25995e9d0..70f23f86b 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -21,7 +21,7 @@ import os.path import pytest_bdd as bdd -from PyQt5.QtSql import QSqlDatabase, QSqlQuery +from PyQt5.QtSql import QSqlDatabase bdd.scenarios('history.feature') diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 50c632f7d..1877929b7 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -22,8 +22,6 @@ import logging import pytest -import hypothesis -from hypothesis import strategies from PyQt5.QtCore import QUrl from qutebrowser.browser import history @@ -69,7 +67,7 @@ def test_get_recent(hist): hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) hist.add_url(QUrl('http://example.com/'), atime=12345) assert list(hist.get_recent()) == [ - ('http://www.qutebrowser.org/', '', 67890 , False), + ('http://www.qutebrowser.org/', '', 67890, False), ('http://example.com/', '', 12345, False), ] From 6412c88277db61393f99c19315db5df4bd0593b1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 8 Apr 2017 07:13:16 -0400 Subject: [PATCH 062/161] Clean up history module. Eliminate out-of-date docstring and remove an unused signal. --- qutebrowser/browser/history.py | 20 +------------------- tests/unit/browser/webkit/test_history.py | 5 +---- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 1a5614fd6..6cc98270c 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -72,25 +72,8 @@ class Entry: class WebHistory(sql.SqlTable): - """The global history of visited pages. + """The global history of visited pages.""" - This is a little more complex as you'd expect so the history can be read - from disk async while new history is already arriving. - - While reading from disk is still ongoing, the history is saved in - self._temp_history instead, and then inserted into the sql table once - the async read completes. - - All history which is new in this session (rather than read from disk from a - previous browsing session) is also stored in self._new_history. - self._saved_count tracks how many of those entries were already written to - disk, so we can always append to the existing data. - - Signals: - cleared: Emitted after the history is cleared. - """ - - cleared = pyqtSignal() async_read_done = pyqtSignal() def __init__(self, parent=None): @@ -128,7 +111,6 @@ class WebHistory(sql.SqlTable): def _do_clear(self): self.delete_all() - self.cleared.emit() @pyqtSlot(QUrl, QUrl, str) def add_from_tab(self, url, requested_url, title): diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 1877929b7..22db1490a 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -93,10 +93,7 @@ def test_clear(qtbot, tmpdir, hist, mocker): def test_clear_force(qtbot, tmpdir, hist): hist.add_url(QUrl('http://example.com/')) hist.add_url(QUrl('http://www.qutebrowser.org/')) - - with qtbot.waitSignal(hist.cleared): - hist.clear(force=True) - + hist.clear(force=True) assert not len(hist) From 024386d1892bacf3e3de5526077b4a072cea4a4b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 8 Apr 2017 16:20:55 -0400 Subject: [PATCH 063/161] Fail on history file parsing errors. Instead of skipping bad history lines during the import to sql, fail hard. We don't want to delete the user's old history file if we couldn't parse all of the lines. --- qutebrowser/browser/history.py | 8 +++++-- tests/unit/browser/webkit/test_history.py | 26 ++++++++++++++++------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 6cc98270c..52d32c114 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -183,12 +183,16 @@ class WebHistory(sql.SqlTable): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: rows = [] - for line in f: + for (i, line) in enumerate(f): + line = line.strip() + if not line: + continue try: row = self._parse_entry(line.strip()) rows.append(row) except ValueError: - log.init.warning('Skipping history line {}'.format(line)) + raise Exception('Failed to parse line #{} of {}: "{}"' + .format(i, path, line)) self.insert_batch(rows) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 22db1490a..b2642f7c8 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -196,20 +196,16 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): assert default_interface is None -def test_read(hist, tmpdir, caplog): +def test_read(hist, tmpdir): histfile = tmpdir / 'history' + # empty line is deliberate, to test skipping empty lines histfile.write('''12345 http://example.com/ title 12346 http://qutebrowser.org/ 67890 http://example.com/path - xyz http://example.com/bad-timestamp - 12345 - http://example.com/no-timestamp - 68891-r http://example.com/path/other - 68891-r-r http://example.com/double-flag''') + 68891-r http://example.com/path/other ''') - with caplog.at_level(logging.WARNING): - hist.read(str(histfile)) + hist.read(str(histfile)) assert list(hist) == [ ('http://example.com/', 'title', 12345, False), @@ -217,3 +213,17 @@ def test_read(hist, tmpdir, caplog): ('http://example.com/path', '', 67890, False), ('http://example.com/path/other', '', 68891, True) ] + + +@pytest.mark.parametrize('line', [ + 'xyz http://example.com/bad-timestamp', + '12345', + 'http://example.com/no-timestamp', + '68891-r-r http://example.com/double-flag', +]) +def test_read_invalid(hist, tmpdir, line): + histfile = tmpdir / 'history' + histfile.write(line) + + with pytest.raises(Exception): + hist.read(str(histfile)) From f110cf4d53fd22ba191bfe54a97dfbcc385d2fa1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 8 Apr 2017 17:18:20 -0400 Subject: [PATCH 064/161] Fix long hang after importing history. Turns out historyContains was getting called for the webkit backend multiple times when the browser starts. This was calling `url in history`, which was enumerating the entire history as `__contains__` was not defined. --- qutebrowser/browser/history.py | 3 +++ qutebrowser/misc/sql.py | 11 +++++++++++ tests/unit/browser/webkit/test_history.py | 8 ++++++++ tests/unit/misc/test_sql.py | 16 ++++++++++++++++ 4 files changed, 38 insertions(+) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 52d32c114..aef551e5c 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -83,6 +83,9 @@ class WebHistory(sql.SqlTable): def __repr__(self): return utils.get_repr(self, length=len(self)) + def __contains__(self, url): + return self.contains('url', url) + def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, entry.atime, diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 707e0c4ce..9cbf6024b 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -139,6 +139,17 @@ class SqlTable(QObject): rec = result.record() yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + def contains(self, field, value): + """Return whether the table contains the matching item. + + Args: + field: Field to match. + value: Value to check for the given field. + """ + query = run_query("SELECT * FROM {} where {} = ? LIMIT 1" + .format(self._name, field), [value]) + return query.next() + def __len__(self): """Return the count of rows in the table.""" result = run_query("SELECT count(*) FROM {}".format(self._name)) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index b2642f7c8..ec36f7a88 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -63,6 +63,14 @@ def test_len(hist): assert len(hist) == 1 +def test_contains(hist): + hist.add_url(QUrl('http://www.example.com/'), title='Title', atime=12345) + assert 'http://www.example.com/' in hist + assert not 'www.example.com' in hist + assert not 'Title' in hist + assert not 12345 in hist + + def test_get_recent(hist): hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) hist.add_url(QUrl('http://example.com/'), atime=12345) diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 387b5ce46..edc156a55 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -88,6 +88,22 @@ def test_len(): assert len(table) == 3 +def test_contains(): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) + assert table.contains('name', 'one') + assert table.contains('name', 'thirteen') + assert table.contains('val', 9) + assert table.contains('lucky', False) + assert table.contains('lucky', True) + assert not table.contains('name', 'oone') + assert not table.contains('name', 1) + assert not table.contains('name', '*') + assert not table.contains('val', 10) + + def test_delete_all(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) table.insert(['one', 1, False]) From e661fb74468236e5d6af32b1f271ffe941367dbb Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 8 Apr 2017 17:24:11 -0400 Subject: [PATCH 065/161] Fix test_history. History doesn't depend on standarddir anymore, the history file path get passed by app.py. --- tests/unit/browser/webkit/test_history.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index ec36f7a88..d444c166c 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -179,7 +179,6 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): else: assert backend == usertypes.Backend.QtWebEngine - monkeypatch.setattr(history.standarddir, 'data', lambda: str(tmpdir)) monkeypatch.setattr(history.objects, 'backend', backend) history.init(qapp) hist = objreg.get('web-history') From 44080b8ad433c73a35058606de8e9fbdf06874b2 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 9 Apr 2017 07:02:14 -0400 Subject: [PATCH 066/161] Fix flake8 errors in test_history --- tests/unit/browser/webkit/test_history.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index d444c166c..4f1a3362d 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -66,9 +66,9 @@ def test_len(hist): def test_contains(hist): hist.add_url(QUrl('http://www.example.com/'), title='Title', atime=12345) assert 'http://www.example.com/' in hist - assert not 'www.example.com' in hist - assert not 'Title' in hist - assert not 12345 in hist + assert 'www.example.com' not in hist + assert 'Title' not in hist + assert 12345 not in hist def test_get_recent(hist): From 231bbe7c2bec0ff432252baa0ba2c39d650d9834 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 11 Apr 2017 07:52:45 -0400 Subject: [PATCH 067/161] Fix race condition on history tests. Two history end2end tests are failing because sqlite is not flushing to disk in time to be read by the test process. My understanding is that sqlite should take an exclusive lock while writing, so it is difficult to understand why this is happening. This can be fixed by adding a delay, but that seems flaky. I'm fixing it by checking qute://history instead of reading the database file. See: https://github.com/qutebrowser/qutebrowser/pull/2295#issuecomment-292786138 and the following discussion. --- tests/end2end/features/history.feature | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 2d02c518a..3ef602712 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -14,7 +14,7 @@ Feature: Page history Then the history file should contain: http://localhost:(port)/data/numbers/1.txt http://localhost:(port)/data/numbers/2.txt - + Scenario: History item with title When I open data/title.html Then the history file should contain: @@ -26,7 +26,7 @@ Feature: Page history Then the history file should contain: r http://localhost:(port)/redirect-to?url=data/title.html Test title http://localhost:(port)/data/title.html Test title - + Scenario: History item with spaces in URL When I open data/title with spaces.html Then the history file should contain: @@ -36,20 +36,24 @@ Feature: Page history When I open data/äöü.html Then the history file should contain: http://localhost:(port)/data/%C3%A4%C3%B6%C3%BC.html Chäschüechli - + + # The following two tests use qute://history instead of checking the + # history file due to a race condition with sqlite. + # https://github.com/qutebrowser/qutebrowser/pull/2295#issuecomment-292786138 @flaky @qtwebengine_todo: Error page message is not implemented Scenario: History with an error When I run :open file:///does/not/exist And I wait for "Error while loading file:///does/not/exist: Error opening /does/not/exist: *" in the log - Then the history file should contain: - file:///does/not/exist Error loading page: file:///does/not/exist + And I open qute://history + Then the page should contain the plaintext "Error loading page: file:///does/not/exist" @qtwebengine_todo: Error page message is not implemented Scenario: History with a 404 When I open status/404 without waiting And I wait for "Error while loading http://localhost:*/status/404: NOT FOUND" in the log - Then the history file should contain: - http://localhost:(port)/status/404 Error loading page: http://localhost:(port)/status/404 + And I open qute://history + Then the page should contain the plaintext "Error loading page: http://localhost:" + And the page should contain the plaintext "/status/404" Scenario: History with invalid URL When I run :tab-only From a8ed9f1c2f8e85b129166ce58ee07d64698d4547 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 11 Apr 2017 08:12:23 -0400 Subject: [PATCH 068/161] Fix qute://version sql init bug. Calling sql.init() in version.version() would replace the existing sql connection and cause a crash when accessed by opening qute://version. Now version relies on sql already being initted, and app.py inits sql early if the --version arg is given. --- qutebrowser/app.py | 3 +++ qutebrowser/utils/version.py | 4 ---- tests/end2end/features/misc.feature | 8 ++++++++ tests/unit/utils/test_version.py | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index dd73827af..0f95c1c5d 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -83,6 +83,9 @@ def run(args): standarddir.init(args) if args.version: + # we need to init sql to print the sql version + # we can use an in-memory database as we just want to query the version + sql.init('') print(version.version()) sys.exit(usertypes.Exit.ok) diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index faa890152..a408ce1fd 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -326,11 +326,7 @@ def version(): lines += _module_versions() lines += ['pdf.js: {}'.format(_pdfjs_version())] - - # we can use an in-memory database as we just want to query the version - sql.init('') lines += ['sqlite: {}'.format(sql.version())] - sql.close() lines += [ 'SSL: {}'.format(QSslSocket.sslLibraryVersionString()), diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index 279df7808..f6c5ce84a 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -702,3 +702,11 @@ Feature: Various utility commands. And I wait for "Renderer process was killed" in the log And I open data/numbers/3.txt Then no crash should happen + And the following tabs should be open: + - data/numbers/3.txt (active) + + ## Other + + Scenario: Open qute://version + When I open qute://version + Then the page should contain the plaintext "Version info" diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 8c0f16081..c0c731f88 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -809,7 +809,7 @@ def test_chromium_version_unpatched(qapp): (True, False, True, True, False), # unknown Linux distribution ]) def test_version_output(git_commit, frozen, style, with_webkit, - known_distribution, stubs, monkeypatch): + known_distribution, stubs, monkeypatch, init_sql): """Test version.version().""" class FakeWebEngineProfile: def httpUserAgent(self): From 784d9bb043213649988ce7a6f2d0568427b2c428 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 11 Apr 2017 09:01:33 -0400 Subject: [PATCH 069/161] Remove code rendered dead by sql implementation. Vulture exposed the following dead code: - AppendLineParse was only used for reading the history text file, which is now a sql database (and the import code for the old text file is simpler and does not need a complex line parser) - async_read_done is no longer used as importing the history text file is synchronous (and should only happen once) - config._init_key_config is unused as it was moved to keyconf.init --- qutebrowser/browser/history.py | 4 +- qutebrowser/config/config.py | 32 ------------ qutebrowser/misc/lineparser.py | 72 +-------------------------- tests/unit/misc/test_lineparser.py | 80 ------------------------------ 4 files changed, 3 insertions(+), 185 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index aef551e5c..d7753cd03 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -21,7 +21,7 @@ import time -from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl +from PyQt5.QtCore import pyqtSlot, QUrl from qutebrowser.commands import cmdutils from qutebrowser.utils import utils, objreg, log, qtutils, usertypes, message @@ -74,8 +74,6 @@ class WebHistory(sql.SqlTable): """The global history of visited pages.""" - async_read_done = pyqtSignal() - def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index e7e74cc19..1245429c2 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -174,38 +174,6 @@ def _init_main_config(parent=None): return -def _init_key_config(parent): - """Initialize the key config. - - Args: - parent: The parent to use for the KeyConfigParser. - """ - from qutebrowser.config.parsers import keyconf - args = objreg.get('args') - try: - key_config = keyconf.KeyConfigParser(standarddir.config(), 'keys.conf', - args.relaxed_config, - parent=parent) - except (keyconf.KeyConfigError, cmdexc.CommandError, - UnicodeDecodeError) as e: - log.init.exception(e) - errstr = "Error while reading key config:\n" - if e.lineno is not None: - errstr += "In line {}: ".format(e.lineno) - error.handle_fatal_exc(e, args, "Error while reading key config!", - pre_text=errstr) - # We didn't really initialize much so far, so we just quit hard. - sys.exit(usertypes.Exit.err_key_config) - else: - objreg.register('key-config', key_config) - save_manager = objreg.get('save-manager') - filename = os.path.join(standarddir.config(), 'keys.conf') - save_manager.add_saveable( - 'key-config', key_config.save, key_config.config_dirty, - config_opt=('general', 'auto-save-config'), filename=filename, - dirty=key_config.is_dirty) - - def _init_misc(): """Initialize misc. config-related files.""" save_manager = objreg.get('save-manager') diff --git a/qutebrowser/misc/lineparser.py b/qutebrowser/misc/lineparser.py index ea9d100b7..2256d9697 100644 --- a/qutebrowser/misc/lineparser.py +++ b/qutebrowser/misc/lineparser.py @@ -21,7 +21,6 @@ import os import os.path -import itertools import contextlib from PyQt5.QtCore import pyqtSlot, pyqtSignal, QObject @@ -96,7 +95,7 @@ class BaseLineParser(QObject): """ assert self._configfile is not None if self._opened: - raise IOError("Refusing to double-open AppendLineParser.") + raise IOError("Refusing to double-open LineParser.") self._opened = True try: if self._binary: @@ -133,73 +132,6 @@ class BaseLineParser(QObject): raise NotImplementedError -class AppendLineParser(BaseLineParser): - - """LineParser which reads lazily and appends data to existing one. - - Attributes: - _new_data: The data which was added in this session. - """ - - def __init__(self, configdir, fname, *, parent=None): - super().__init__(configdir, fname, binary=False, parent=parent) - self.new_data = [] - self._fileobj = None - - def __iter__(self): - if self._fileobj is None: - raise ValueError("Iterating without open() being called!") - file_iter = (line.rstrip('\n') for line in self._fileobj) - return itertools.chain(file_iter, iter(self.new_data)) - - @contextlib.contextmanager - def open(self): - """Open the on-disk history file. Needed for __iter__.""" - try: - with self._open('r') as f: - self._fileobj = f - yield - except FileNotFoundError: - self._fileobj = [] - yield - finally: - self._fileobj = None - - def get_recent(self, count=4096): - """Get the last count bytes from the underlying file.""" - with self._open('r') as f: - f.seek(0, os.SEEK_END) - size = f.tell() - try: - if size - count > 0: - offset = size - count - else: - offset = 0 - f.seek(offset) - data = f.readlines() - finally: - f.seek(0, os.SEEK_END) - return data - - def save(self): - do_save = self._prepare_save() - if not do_save: - return - with self._open('a') as f: - self._write(f, self.new_data) - self.new_data = [] - self._after_save() - - def clear(self): - do_save = self._prepare_save() - if not do_save: - return - with self._open('w'): - pass - self.new_data = [] - self._after_save() - - class LineParser(BaseLineParser): """Parser for configuration files which are simply line-based. @@ -240,7 +172,7 @@ class LineParser(BaseLineParser): def save(self): """Save the config file.""" if self._opened: - raise IOError("Refusing to double-open AppendLineParser.") + raise IOError("Refusing to double-open LineParser.") do_save = self._prepare_save() if not do_save: return diff --git a/tests/unit/misc/test_lineparser.py b/tests/unit/misc/test_lineparser.py index af439c006..adae2485d 100644 --- a/tests/unit/misc/test_lineparser.py +++ b/tests/unit/misc/test_lineparser.py @@ -125,83 +125,3 @@ class TestLineParser: lineparser._prepare_save = lambda: False lineparser.save() assert (tmpdir / 'file').read() == 'pristine\n' - - -class TestAppendLineParser: - - BASE_DATA = ['old data 1', 'old data 2'] - - @pytest.fixture - def lineparser(self, tmpdir): - """Fixture to get an AppendLineParser for tests.""" - lp = lineparsermod.AppendLineParser(str(tmpdir), 'file') - lp.new_data = self.BASE_DATA - lp.save() - return lp - - def _get_expected(self, new_data): - """Get the expected data with newlines.""" - return '\n'.join(self.BASE_DATA + new_data) + '\n' - - def test_save(self, tmpdir, lineparser): - """Test save().""" - new_data = ['new data 1', 'new data 2'] - lineparser.new_data = new_data - lineparser.save() - assert (tmpdir / 'file').read() == self._get_expected(new_data) - - def test_clear(self, tmpdir, lineparser): - """Check if calling clear() empties both pending and persisted data.""" - lineparser.new_data = ['one', 'two'] - lineparser.save() - assert (tmpdir / 'file').read() == "old data 1\nold data 2\none\ntwo\n" - - lineparser.new_data = ['one', 'two'] - lineparser.clear() - lineparser.save() - assert not lineparser.new_data - assert (tmpdir / 'file').read() == "" - - def test_iter_without_open(self, lineparser): - """Test __iter__ without having called open().""" - with pytest.raises(ValueError): - iter(lineparser) - - def test_iter(self, lineparser): - """Test __iter__.""" - new_data = ['new data 1', 'new data 2'] - lineparser.new_data = new_data - with lineparser.open(): - assert list(lineparser) == self.BASE_DATA + new_data - - def test_iter_not_found(self, mocker): - """Test __iter__ with no file.""" - open_mock = mocker.patch( - 'qutebrowser.misc.lineparser.AppendLineParser._open') - open_mock.side_effect = FileNotFoundError - new_data = ['new data 1', 'new data 2'] - linep = lineparsermod.AppendLineParser('foo', 'bar') - linep.new_data = new_data - with linep.open(): - assert list(linep) == new_data - - def test_get_recent_none(self, tmpdir): - """Test get_recent with no data.""" - (tmpdir / 'file2').ensure() - linep = lineparsermod.AppendLineParser(str(tmpdir), 'file2') - assert linep.get_recent() == [] - - def test_get_recent_little(self, lineparser): - """Test get_recent with little data.""" - data = [e + '\n' for e in self.BASE_DATA] - assert lineparser.get_recent() == data - - def test_get_recent_much(self, lineparser): - """Test get_recent with much data.""" - size = 64 - new_data = ['new data {}'.format(i) for i in range(size)] - lineparser.new_data = new_data - lineparser.save() - data = os.linesep.join(self.BASE_DATA + new_data) + os.linesep - data = [e + '\n' for e in data[-size:].splitlines()] - assert lineparser.get_recent(size) == data From 9d4888a7729fcd6f573b23d081c12e2f4bf91de1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 15 Apr 2017 23:01:24 -0400 Subject: [PATCH 070/161] Optimize qute://history for SQL backend. The old implementation was looping through the whole history list, which for SQL was selecting every row in the database. The history benchmark was taking ~2s. If this is rewritten as a specialized SQL query, the benchmark takes ~10ms, an order of magnitude faster than the original non-SQL implementation. --- qutebrowser/browser/history.py | 20 ++++++- qutebrowser/browser/qutescheme.py | 76 +++------------------------ tests/unit/browser/test_qutescheme.py | 13 +++-- 3 files changed, 33 insertions(+), 76 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index d7753cd03..dc22d37a8 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -86,13 +86,31 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" - self.insert([entry.url_str(), entry.title, entry.atime, + self.insert([entry.url_str(), entry.title, int(entry.atime), entry.redirect]) def get_recent(self): """Get the most recent history entries.""" return self.select(sort_by='atime', sort_order='desc', limit=100) + def entries_between(self, earliest, latest): + """Iterate non-redirect, non-qute entries between two timestamps. + + Args: + earliest: Omit timestamps earlier than this. + latest: Omit timestamps later than this. + """ + result = sql.run_query('SELECT * FROM History ' + 'where not redirect ' + 'and not url like "qute://%" ' + 'and atime > {} ' + 'and atime <= {} ' + 'ORDER BY atime desc' + .format(earliest, latest)) + while result.next(): + rec = result.record() + yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + @cmdutils.register(name='history-clear', instance='web-history') def clear(self, force=False): """Clear all browsing history. diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index d12fb8f8e..55a16801a 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -186,81 +186,17 @@ def qute_bookmarks(_url): return 'text/html', html -def history_data(start_time): # noqa +def history_data(start_time): """Return history data Arguments: start_time -- select history starting from this timestamp. """ - def history_iter(start_time, reverse=False): - """Iterate through the history and get items we're interested. - - Arguments: - reverse -- whether to reverse the history_dict before iterating. - """ - history = list(objreg.get('web-history')) - if reverse: - history = reversed(history) - - # when history_dict is not reversed, we need to keep track of last item - # so that we can yield its atime - last_item = None - - # end is 24hrs earlier than start - end_time = start_time - 24*60*60 - - for item in history: - # Skip redirects - # Skip qute:// links - if item.redirect or item.url.startswith('qute://'): - continue - - # Skip items out of time window - item_newer = item.atime > start_time - item_older = item.atime <= end_time - if reverse: - # history_dict is reversed, we are going back in history. - # so: - # abort if item is older than start_time+24hr - # skip if item is newer than start - if item_older: - yield {"next": int(item.atime)} - return - if item_newer: - continue - else: - # history_dict isn't reversed, we are going forward in history. - # so: - # abort if item is newer than start_time - # skip if item is older than start_time+24hrs - if item_older: - last_item = item - continue - if item_newer: - yield {"next": int(last_item.atime if last_item else -1)} - return - - # Use item's url as title if there's no title. - item_title = item.title if item.title else item.url - item_time = int(item.atime * 1000) - - yield {"url": item.url, "title": item_title, "time": item_time} - - # if we reached here, we had reached the end of history - yield {"next": int(last_item.atime if last_item else -1)} - - if sys.hexversion >= 0x03050000: - # On Python >= 3.5 we can reverse the ordereddict in-place and thus - # apply an additional performance improvement in history_iter. - # On my machine, this gets us down from 550ms to 72us with 500k old - # items. - history = history_iter(start_time, reverse=True) - else: - # On Python 3.4, we can't do that, so we'd need to copy the entire - # history to a list. There, filter first and then reverse it here. - history = reversed(list(history_iter(start_time, reverse=False))) - - return list(history) + # end is 24hrs earlier than start + end_time = start_time - 24*60*60 + entries = objreg.get('web-history').entries_between(end_time, start_time) + return [{"url": e.url, "title": e.title or e.url, "time": e.atime * 1000} + for e in entries] @add_handler('history') diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 86a322322..0f9f43373 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -132,6 +132,7 @@ class TestHistoryHandler: assert item['time'] <= start_time * 1000 assert item['time'] > end_time * 1000 + @pytest.mark.skip("TODO: do we need next?") @pytest.mark.parametrize("start_time_offset, next_time", [ (0, 24*60*60), (24*60*60, 48*60*60), @@ -153,14 +154,16 @@ class TestHistoryHandler: assert items[0]["next"] == now - next_time def test_qute_history_benchmark(self, fake_web_history, benchmark, now): - # items must be earliest-first to ensure history is sorted properly - for t in range(100000, 0, -1): # one history per second - entry = history.Entry( + entries = [] + for t in range(100000): # one history per second + entry = fake_web_history.Entry( atime=str(now - t), url=QUrl('www.x.com/{}'.format(t)), - title='x at {}'.format(t)) - fake_web_history._add_entry(entry) + title='x at {}'.format(t), + redirect=False) + entries.append(entry) + fake_web_history.insert_batch(entries) url = QUrl("qute://history/data?start_time={}".format(now)) _mimetype, data = benchmark(qutescheme.qute_history, url) assert len(json.loads(data)) > 1 From 71191f10a28f86b692864bba6679d9525207c7eb Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 18 Apr 2017 07:48:11 -0400 Subject: [PATCH 071/161] Only complete most recent atime for url. The history completion query is extended to pick only the most recent item for a given url. The tests in test_models now check for ordering of elements. --- qutebrowser/browser/qutescheme.py | 2 +- qutebrowser/completion/models/sqlcategory.py | 6 +- qutebrowser/completion/models/urlmodel.py | 4 +- tests/helpers/fixtures.py | 2 +- tests/helpers/stubs.py | 18 --- tests/unit/completion/test_models.py | 121 ++++++++++++++----- tests/unit/completion/test_sqlcategory.py | 10 ++ 7 files changed, 107 insertions(+), 56 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 55a16801a..ba8486ff4 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -187,7 +187,7 @@ def qute_bookmarks(_url): def history_data(start_time): - """Return history data + """Return history data. Arguments: start_time -- select history starting from this timestamp. diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 693aaa841..9f78163ca 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -30,7 +30,7 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, sort_by=None, sort_order=None, select='*', - where=None, parent=None): + where=None, group_by=None, parent=None): """Create a new completion category backed by a sql table. Args: @@ -46,6 +46,7 @@ class SqlCategory(QSqlQueryModel): self._sort_order = sort_order self._select = select self._where = where + self._group_by = group_by self.set_pattern('', [0]) def set_pattern(self, pattern, columns_to_filter): @@ -67,6 +68,9 @@ class SqlCategory(QSqlQueryModel): if self._where: querystr += ' and ' + self._where + if self._group_by: + querystr += ' group by {}'.format(self._group_by) + if self._sort_by: assert self._sort_order in ['asc', 'desc'] querystr += ' order by {} {}'.format(self._sort_by, diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index ea3f3f610..9e601cd39 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -74,9 +74,9 @@ def url(): model.add_category(listcategory.ListCategory('Bookmarks', bookmarks)) timefmt = config.get('completion', 'timestamp-format') - select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) + select_time = "strftime('{}', max(atime), 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'History', sort_order='desc', sort_by='atime', + 'History', sort_order='desc', sort_by='atime', group_by='url', select='url, title, {}'.format(select_time), where='not redirect') model.add_category(hist_cat) return model diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 24087ed18..1a3dfd8ea 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -259,7 +259,7 @@ def bookmark_manager_stub(stubs): @pytest.fixture def session_manager_stub(stubs): - """Fixture which provides a fake web-history object.""" + """Fixture which provides a fake session-manager object.""" stub = stubs.SessionManagerStub() objreg.register('session-manager', stub) yield stub diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 8b2a235d3..05341103c 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -518,24 +518,6 @@ class QuickmarkManagerStub(UrlMarkManagerStub): self.delete(key) -class WebHistoryStub(QObject): - - """Stub for the web-history object.""" - - add_completion_item = pyqtSignal(history.Entry) - cleared = pyqtSignal() - - def __init__(self, parent=None): - super().__init__(parent) - self.history_dict = collections.OrderedDict() - - def __iter__(self): - return iter(self.history_dict.values()) - - def __len__(self): - return len(self.history_dict) - - class HostBlockerStub: """Stub for the host-blocker object.""" diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index e071f1584..cea9d0ccd 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -43,18 +43,18 @@ def _check_completions(model, expected): ... } """ + actual = {} assert model.rowCount() == len(expected) for i in range(0, model.rowCount()): catidx = model.index(i, 0) catname = model.data(catidx) - assert catname in expected - expected_cat = expected[catname] - assert model.rowCount(catidx) == len(expected_cat) + actual[catname] = [] for j in range(model.rowCount(catidx)): name = model.data(model.index(j, 0, parent=catidx)) desc = model.data(model.index(j, 1, parent=catidx)) misc = model.data(model.index(j, 2, parent=catidx)) - assert (name, desc, misc) in expected_cat + actual[catname].append((name, desc, misc)) + assert actual == expected # sanity-check the column_widths assert len(model.column_widths) == 3 assert sum(model.column_widths) == 100 @@ -155,18 +155,26 @@ def bookmarks(bookmark_manager_stub): @pytest.fixture -def web_history(stubs, init_sql): +def web_history_stub(stubs, init_sql): + return sql.SqlTable("History", ['url', 'title', 'atime', 'redirect']) + + +@pytest.fixture +def web_history(web_history_stub, init_sql): """Pre-populate the web-history database.""" - table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect']) - table.insert(['http://some-redirect.example.com', 'redirect', - datetime(2016, 9, 5).timestamp(), True]) - table.insert(['http://qutebrowser.org', 'qutebrowser', - datetime(2015, 9, 5).timestamp(), False]) - table.insert(['https://python.org', 'Welcome to Python.org', - datetime(2016, 3, 8).timestamp(), False]) - table.insert(['https://github.com', 'https://github.com', - datetime(2016, 5, 1).timestamp(), False]) - return table + web_history_stub.insert(['http://some-redirect.example.com', 'redirect', + datetime(2016, 9, 5).timestamp(), True]) + web_history_stub.insert(['http://qutebrowser.org', 'qutebrowser', + datetime(2015, 9, 5).timestamp(), False]) + web_history_stub.insert(['https://python.org', 'Welcome to Python.org', + datetime(2016, 2, 8).timestamp(), False]) + web_history_stub.insert(['https://python.org', 'Welcome to Python.org', + datetime(2016, 3, 8).timestamp(), False]) + web_history_stub.insert(['https://python.org', 'Welcome to Python.org', + datetime(2014, 3, 8).timestamp(), False]) + web_history_stub.insert(['https://github.com', 'https://github.com', + datetime(2016, 5, 1).timestamp(), False]) + return web_history_stub def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, @@ -186,15 +194,16 @@ def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, 'rr': 'roll', 'ro': 'rock'}) model = miscmodels.command() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Commands": [ - ('stop', 'stop qutebrowser', 's'), ('drop', 'drop all user data', ''), - ('roll', 'never gonna give you up', 'rr'), ('rock', "Alias for 'roll'", 'ro'), + ('roll', 'never gonna give you up', 'rr'), + ('stop', 'stop qutebrowser', 's'), ] }) @@ -214,23 +223,24 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): _patch_cmdutils(monkeypatch, stubs, module + '.cmdutils') _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') model = miscmodels.helptopic() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Commands": [ - (':stop', 'stop qutebrowser', 's'), (':drop', 'drop all user data', ''), - (':roll', 'never gonna give you up', 'rr'), (':hide', '', ''), + (':roll', 'never gonna give you up', 'rr'), + (':stop', 'stop qutebrowser', 's'), ], "Settings": [ ('general->time', 'Is an illusion.', None), ('general->volume', 'Goes to 11', None), + ('searchengines->DEFAULT', '', None), ('ui->gesture', 'Waggle your hands to control qutebrowser', None), ('ui->mind', 'Enable mind-control ui (experimental)', None), ('ui->voice', 'Whether to respond to voice commands', None), - ('searchengines->DEFAULT', '', None), ] }) @@ -238,6 +248,7 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -253,14 +264,15 @@ def test_quickmark_completion(qtmodeltester, quickmarks): def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Bookmarks": [ + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ('https://github.com', 'GitHub', None), ('https://python.org', 'Welcome to Python.org', None), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ] }) @@ -273,22 +285,24 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, - quickmarks, bookmarks, and urls are included - entries are sorted by access time - redirect entries are not included + - only the most recent entry is included for each url """ config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Quickmarks": [ - ('https://wiki.archlinux.org', 'aw', None), ('https://duckduckgo.com', 'ddg', None), + ('https://wiki.archlinux.org', 'aw', None), ('https://wikipedia.org', 'wiki', None), ], "Bookmarks": [ + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ('https://github.com', 'GitHub', None), ('https://python.org', 'Welcome to Python.org', None), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ], "History": [ ('https://github.com', 'https://github.com', '2016-05-01'), @@ -298,17 +312,48 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, }) +@pytest.mark.parametrize('url, title, pattern, rowcount', [ + ('example.com', 'Site Title', '', 1), + ('example.com', 'Site Title', 'ex', 1), + ('example.com', 'Site Title', 'am', 1), + ('example.com', 'Site Title', 'com', 1), + ('example.com', 'Site Title', 'ex com', 1), + ('example.com', 'Site Title', 'com ex', 0), + ('example.com', 'Site Title', 'ex foo', 0), + ('example.com', 'Site Title', 'foo com', 0), + ('example.com', 'Site Title', 'exm', 0), + ('example.com', 'Site Title', 'Si Ti', 1), + ('example.com', 'Site Title', 'Ti Si', 0), + ('example.com', '', 'foo', 0), + ('foo_bar', '', '_', 1), + ('foobar', '', '_', 0), + ('foo%bar', '', '%', 1), + ('foobar', '', '%', 0), +]) +def test_url_completion_pattern(config_stub, web_history_stub, + quickmark_manager_stub, bookmark_manager_stub, + url, title, pattern, rowcount): + """Test that url completion filters by url and title.""" + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} + web_history_stub.insert([url, title, 0, False]) + model = urlmodel.url() + model.set_pattern(pattern) + # 2, 0 is History + assert model.rowCount(model.index(2, 0)) == rowcount + + def test_url_completion_delete_bookmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - # delete item (1, 0) -> (bookmarks, 'https://github.com' ) - view = _mock_view_index(model, 1, 0, qtbot) + # delete item (1, 1) -> (bookmarks, 'https://github.com') + view = _mock_view_index(model, 1, 1, qtbot) model.delete_cur_item(view) assert 'https://github.com' not in bookmarks.marks assert 'https://python.org' in bookmarks.marks @@ -321,11 +366,12 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - # delete item (0, 1) -> (quickmarks, 'ddg' ) - view = _mock_view_index(model, 0, 1, qtbot) + # delete item (0, 0) -> (quickmarks, 'ddg' ) + view = _mock_view_index(model, 0, 0, qtbot) model.delete_cur_item(view) assert 'aw' in quickmarks.marks assert 'ddg' not in quickmarks.marks @@ -338,6 +384,7 @@ def test_url_completion_delete_history(qtmodeltester, config_stub, """Test that deleting a history entry is a noop.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -350,13 +397,14 @@ def test_url_completion_delete_history(qtmodeltester, config_stub, def test_session_completion(qtmodeltester, session_manager_stub): session_manager_stub.sessions = ['default', '1', '2'] model = miscmodels.session() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { - "Sessions": [('default', None, None), - ('1', None, None), - ('2', None, None)] + "Sessions": [('1', None, None), + ('2', None, None), + ('default', None, None)] }) @@ -371,6 +419,7 @@ def test_tab_completion(qtmodeltester, fake_web_tab, app_stub, win_registry, fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0), ] model = miscmodels.buffer() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -398,6 +447,7 @@ def test_tab_completion_delete(qtmodeltester, fake_web_tab, qtbot, app_stub, fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0), ] model = miscmodels.buffer() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -415,14 +465,15 @@ def test_setting_section_completion(qtmodeltester, monkeypatch, stubs): _patch_config_section_desc(monkeypatch, stubs, module + '.configdata.SECTION_DESC') model = configmodel.section() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Sections": [ ('general', 'General/miscellaneous options.', None), - ('ui', 'General options related to the user interface.', None), ('searchengines', 'Definitions of search engines ...', None), + ('ui', 'General options related to the user interface.', None), ] }) @@ -435,6 +486,7 @@ def test_setting_option_completion(qtmodeltester, monkeypatch, stubs, 'mind': 'on', 'voice': 'sometimes'}} model = configmodel.option('ui') + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -457,6 +509,7 @@ def test_setting_option_completion_valuelist(qtmodeltester, monkeypatch, stubs, } } model = configmodel.option('searchengines') + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -471,6 +524,7 @@ def test_setting_value_completion(qtmodeltester, monkeypatch, stubs, _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') config_stub.data = {'general': {'volume': '0'}} model = configmodel.value('general', 'volume') + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -503,16 +557,17 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, 'rr': 'roll', 'ro': 'rock'}) model = miscmodels.bind('s') + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Commands": [ - ('stop', 'stop qutebrowser', 's'), ('drop', 'drop all user data', ''), ('hide', '', ''), - ('roll', 'never gonna give you up', 'rr'), ('rock', "Alias for 'roll'", 'ro'), + ('roll', 'never gonna give you up', 'rr'), + ('stop', 'stop qutebrowser', 's'), ] }) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 03288f016..f29589717 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -149,6 +149,16 @@ def test_where(): _validate(cat, [('foo', 'bar', False)]) +def test_group(): + table = sql.SqlTable('Foo', ['a', 'b']) + table.insert(['foo', 1]) + table.insert(['bar', 3]) + table.insert(['foo', 2]) + table.insert(['bar', 0]) + cat = sqlcategory.SqlCategory('Foo', select='a, max(b)', group_by='a') + _validate(cat, [('bar', 3), ('foo', 2)]) + + def test_entry(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) assert hasattr(table.Entry, 'a') From 0aa0478327246e68620f2326cefd9ca9ecc02785 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 7 May 2017 11:06:33 -0400 Subject: [PATCH 072/161] Use EXISTS for SqlTable.contains. --- qutebrowser/misc/sql.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 9cbf6024b..21af483c9 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -146,9 +146,10 @@ class SqlTable(QObject): field: Field to match. value: Value to check for the given field. """ - query = run_query("SELECT * FROM {} where {} = ? LIMIT 1" + query = run_query("Select EXISTS(SELECT * FROM {} where {} = ?)" .format(self._name, field), [value]) - return query.next() + query.next() + return query.value(0) def __len__(self): """Return the count of rows in the table.""" From eb6126906899e5d957bd55adf915468e2ab25c2f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 7 May 2017 22:21:46 -0400 Subject: [PATCH 073/161] Fix qute://history javascript for SQL. Returning "next" was no longer possible as the SQL query does not fetch more items than necessary. This is solved by using a start time, a limit, and an offset. The offset is needed to prevent fetching duplicate items if multiple entries have the same timestamp. Two of the history tests that relied on qute://history were changed to rely on qute://history/data instead to make them less failure-prone. --- qutebrowser/browser/history.py | 19 +++++++++++++ qutebrowser/browser/qutescheme.py | 26 ++++++++++++----- qutebrowser/javascript/history.js | 34 +++++++++++++++-------- tests/end2end/features/history.feature | 4 +-- tests/unit/browser/webkit/test_history.py | 27 ++++++++++++++++++ 5 files changed, 90 insertions(+), 20 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index dc22d37a8..2f7040c21 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -111,6 +111,25 @@ class WebHistory(sql.SqlTable): rec = result.record() yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + def entries_before(self, latest, limit, offset): + """Iterate non-redirect, non-qute entries occuring before a timestamp. + + Args: + latest: Omit timestamps more recent than this. + limit: Max number of entries to include. + offset: Number of entries to skip. + """ + result = sql.run_query('SELECT * FROM History ' + 'where not redirect ' + 'and not url like "qute://%" ' + 'and atime <= {} ' + 'ORDER BY atime desc ' + 'limit {} offset {}' + .format(latest, limit, offset)) + while result.next(): + rec = result.record() + yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + @cmdutils.register(name='history-clear', instance='web-history') def clear(self, force=False): """Clear all browsing history. diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index ba8486ff4..6039fe55e 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -186,15 +186,22 @@ def qute_bookmarks(_url): return 'text/html', html -def history_data(start_time): +def history_data(start_time, offset=None): """Return history data. Arguments: - start_time -- select history starting from this timestamp. + start_time: select history starting from this timestamp. + offset: number of items to skip """ - # end is 24hrs earlier than start - end_time = start_time - 24*60*60 - entries = objreg.get('web-history').entries_between(end_time, start_time) + hist = objreg.get('web-history') + if offset is not None: + # end is 24hrs earlier than start + entries = hist.entries_before(start_time, limit=1000, offset=offset) + else: + # end is 24hrs earlier than start + end_time = start_time - 24*60*60 + entries = hist.entries_between(end_time, start_time) + return [{"url": e.url, "title": e.title or e.url, "time": e.atime * 1000} for e in entries] @@ -203,6 +210,11 @@ def history_data(start_time): def qute_history(url): """Handler for qute://history. Display and serve history.""" if url.path() == '/data': + try: + offset = QUrlQuery(url).queryItemValue("offset") + offset = int(offset) if offset else None + except ValueError as e: + raise QuteSchemeError("Query parameter start_time is invalid", e) # Use start_time in query or current time. try: start_time = QUrlQuery(url).queryItemValue("start_time") @@ -210,7 +222,7 @@ def qute_history(url): except ValueError as e: raise QuteSchemeError("Query parameter start_time is invalid", e) - return 'text/html', json.dumps(history_data(start_time)) + return 'text/html', json.dumps(history_data(start_time, offset)) else: if ( config.get('content', 'allow-javascript') and @@ -244,7 +256,7 @@ def qute_history(url): (i["url"], i["title"], datetime.datetime.fromtimestamp(i["time"]/1000), QUrl(i["url"]).host()) - for i in history_data(start_time) if "next" not in i + for i in history_data(start_time) ] return 'text/html', jinja.render( diff --git a/qutebrowser/javascript/history.js b/qutebrowser/javascript/history.js index f46ceb49d..a6cf1d7aa 100644 --- a/qutebrowser/javascript/history.js +++ b/qutebrowser/javascript/history.js @@ -23,8 +23,12 @@ window.loadHistory = (function() { // Date of last seen item. var lastItemDate = null; - // The time to load next. + // Each request for new items includes the time of the last item and an + // offset. The offset is equal to the number of items from the previous + // request that had time=nextTime, and causes the next request to skip + // those items to avoid duplicates. var nextTime = null; + var nextOffset = 0; // The URL to fetch data from. var DATA_URL = "qute://history/data"; @@ -157,7 +161,15 @@ window.loadHistory = (function() { return; } - for (var i = 0, len = history.length - 1; i < len; i++) { + if (history.length == 0) { + // Reached end of history + window.onscroll = null; + EOF_MESSAGE.style.display = "block"; + LOAD_LINK.style.display = "none"; + return + } + + for (var i = 0, len = history.length; i < len; i++) { var item = history[i]; var currentItemDate = new Date(item.time); getSessionNode(currentItemDate).appendChild(makeHistoryRow( @@ -166,14 +178,12 @@ window.loadHistory = (function() { lastItemDate = currentItemDate; } - var next = history[history.length - 1].next; - if (next === -1) { - // Reached end of history - window.onscroll = null; - EOF_MESSAGE.style.display = "block"; - LOAD_LINK.style.display = "none"; - } else { - nextTime = next; + nextTime = history[history.length - 1].time + nextOffset = 0; + for (var i = history.length - 1; i >= 0; i--) { + if (history[i].time == nextTime) { + nextOffset++; + } } } @@ -183,9 +193,11 @@ window.loadHistory = (function() { */ function loadHistory() { if (nextTime === null) { - getJSON(DATA_URL, receiveHistory); + var url = DATA_URL.concat("?offset=0"); + getJSON(url, receiveHistory); } else { var url = DATA_URL.concat("?start_time=", nextTime.toString()); + var url = url.concat("&offset=", nextOffset.toString()); getJSON(url, receiveHistory); } } diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 3ef602712..6efd9ad0e 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -44,14 +44,14 @@ Feature: Page history Scenario: History with an error When I run :open file:///does/not/exist And I wait for "Error while loading file:///does/not/exist: Error opening /does/not/exist: *" in the log - And I open qute://history + And I open qute://history/data Then the page should contain the plaintext "Error loading page: file:///does/not/exist" @qtwebengine_todo: Error page message is not implemented Scenario: History with a 404 When I open status/404 without waiting And I wait for "Error while loading http://localhost:*/status/404: NOT FOUND" in the log - And I open qute://history + And I open qute://history/data Then the page should contain the plaintext "Error loading page: http://localhost:" And the page should contain the plaintext "/status/404" diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 4f1a3362d..2118a6c84 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -80,6 +80,33 @@ def test_get_recent(hist): ] +def test_entries_between(hist): + hist.add_url(QUrl('http://www.example.com/1'), atime=12345) + hist.add_url(QUrl('http://www.example.com/2'), atime=12346) + hist.add_url(QUrl('http://www.example.com/3'), atime=12347) + hist.add_url(QUrl('http://www.example.com/4'), atime=12348) + hist.add_url(QUrl('http://www.example.com/5'), atime=12348) + hist.add_url(QUrl('http://www.example.com/6'), atime=12349) + hist.add_url(QUrl('http://www.example.com/7'), atime=12350) + + times = [x.atime for x in hist.entries_between(12346, 12349)] + assert times == [12349, 12348, 12348, 12347] + + +def test_entries_before(hist): + hist.add_url(QUrl('http://www.example.com/1'), atime=12346) + hist.add_url(QUrl('http://www.example.com/2'), atime=12346) + hist.add_url(QUrl('http://www.example.com/3'), atime=12347) + hist.add_url(QUrl('http://www.example.com/4'), atime=12348) + hist.add_url(QUrl('http://www.example.com/5'), atime=12348) + hist.add_url(QUrl('http://www.example.com/6'), atime=12348) + hist.add_url(QUrl('http://www.example.com/7'), atime=12349) + hist.add_url(QUrl('http://www.example.com/8'), atime=12349) + + times = [x.atime for x in hist.entries_before(12348, limit=3, offset=2)] + assert times == [12348, 12347, 12346] + + def test_save(tmpdir, hist): hist.add_url(QUrl('http://example.com/'), atime=12345) hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) From 882da71397e30d932269687254cb4cd7b60a6a7c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 9 May 2017 08:01:31 -0400 Subject: [PATCH 074/161] Remove unused imports --- qutebrowser/browser/qutescheme.py | 1 - tests/helpers/stubs.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 6039fe55e..4ebc626d3 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -26,7 +26,6 @@ Module attributes: import json import os -import sys import time import urllib.parse import datetime diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 05341103c..b852426d2 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -29,7 +29,7 @@ from PyQt5.QtNetwork import (QNetworkRequest, QAbstractNetworkCache, QNetworkCacheMetaData) from PyQt5.QtWidgets import QCommonStyle, QLineEdit, QWidget, QTabBar -from qutebrowser.browser import browsertab, history +from qutebrowser.browser import browsertab from qutebrowser.config import configexc from qutebrowser.utils import usertypes, utils from qutebrowser.mainwindow import mainwindow From 119c33ac32afc4eabe21ecc2fc79b06730c91855 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 9 May 2017 12:02:59 -0400 Subject: [PATCH 075/161] Remove base.py from check_coverage. This module no longer exists. --- scripts/dev/check_coverage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index a2950b658..c01ae64af 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -156,8 +156,6 @@ PERFECT_FILES = [ ('tests/unit/utils/test_javascript.py', 'utils/javascript.py'), - ('tests/unit/completion/test_models.py', - 'completion/models/base.py'), ('tests/unit/completion/test_models.py', 'completion/models/urlmodel.py'), ('tests/unit/completion/test_sqlcategory.py', From 9b25b7ee5df8477e1d179aa2a81e6fdc6e1abca1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 9 May 2017 12:08:31 -0400 Subject: [PATCH 076/161] Fix misspelling of 'occurs' --- qutebrowser/browser/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 2f7040c21..89f850b9d 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -112,7 +112,7 @@ class WebHistory(sql.SqlTable): yield self.Entry(*[rec.value(i) for i in range(rec.count())]) def entries_before(self, latest, limit, offset): - """Iterate non-redirect, non-qute entries occuring before a timestamp. + """Iterate non-redirect, non-qute entries occurring before a timestamp. Args: latest: Omit timestamps more recent than this. From e201a42383891cc5142759536aabc72782a663c0 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 9 May 2017 21:18:15 -0400 Subject: [PATCH 077/161] Fix eslint errors --- qutebrowser/javascript/history.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/qutebrowser/javascript/history.js b/qutebrowser/javascript/history.js index a6cf1d7aa..3f5f9def6 100644 --- a/qutebrowser/javascript/history.js +++ b/qutebrowser/javascript/history.js @@ -161,14 +161,17 @@ window.loadHistory = (function() { return; } - if (history.length == 0) { + if (history.length === 0) { // Reached end of history window.onscroll = null; EOF_MESSAGE.style.display = "block"; LOAD_LINK.style.display = "none"; - return + return; } + nextTime = history[history.length - 1].time; + nextOffset = 0; + for (var i = 0, len = history.length; i < len; i++) { var item = history[i]; var currentItemDate = new Date(item.time); @@ -176,12 +179,7 @@ window.loadHistory = (function() { item.url, item.title, currentItemDate.toLocaleTimeString() )); lastItemDate = currentItemDate; - } - - nextTime = history[history.length - 1].time - nextOffset = 0; - for (var i = history.length - 1; i >= 0; i--) { - if (history[i].time == nextTime) { + if (item.time === nextTime) { nextOffset++; } } @@ -192,12 +190,11 @@ window.loadHistory = (function() { * @return {void} */ function loadHistory() { + var url = DATA_URL.concat("?offset=", nextOffset.toString()); if (nextTime === null) { - var url = DATA_URL.concat("?offset=0"); getJSON(url, receiveHistory); } else { - var url = DATA_URL.concat("?start_time=", nextTime.toString()); - var url = url.concat("&offset=", nextOffset.toString()); + url = url.concat("&start_time=", nextTime.toString()); getJSON(url, receiveHistory); } } From 87643040a41dd4c2a3af17d6d781a355b41cfa62 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 11 May 2017 08:20:37 -0400 Subject: [PATCH 078/161] Fix test_history for python < 3.6. Mock.assert_called is only in python 3.6. For earlier versions we must use `assert m.called`. Weird errors only appearing in CI, trying to debug... --- tests/unit/browser/webkit/test_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 2118a6c84..e6bbfcf69 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -122,7 +122,7 @@ def test_clear(qtbot, tmpdir, hist, mocker): m = mocker.patch('qutebrowser.browser.history.message.confirm_async') hist.clear() - m.assert_called() + assert m.called def test_clear_force(qtbot, tmpdir, hist): From 20000088dee5532c280fd741d665d3ab010f7a56 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 13 May 2017 21:32:47 -0400 Subject: [PATCH 079/161] Add debug-dump-history and fix sql history tests. Trying to read from the sql database from another process was flaky. This adds a debug-dump-history command which is used by the history BDD tests to validate the history contents. It outputs history in the old pre-SQL text format, so it might be useful for those who want to manipulate their history as text. --- qutebrowser/browser/history.py | 22 ++++++++++++ qutebrowser/misc/sql.py | 10 ++++-- tests/end2end/features/history.feature | 28 +++++++-------- tests/end2end/features/test_history_bdd.py | 41 +++++++++++----------- tests/unit/browser/webkit/test_history.py | 15 ++++++++ tests/unit/misc/test_sql.py | 3 +- 6 files changed, 79 insertions(+), 40 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 89f850b9d..e19463e75 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -19,6 +19,7 @@ """Simple history which gets written to disk.""" +import os import time from PyQt5.QtCore import pyqtSlot, QUrl @@ -233,6 +234,27 @@ class WebHistory(sql.SqlTable): .format(i, path, line)) self.insert_batch(rows) + @cmdutils.register(instance='web-history', debug=True) + def debug_dump_history(self, dest): + """Dump the history to a file in the old pre-SQL format. + + Args: + dest: Where to write the file to. + """ + dest = os.path.expanduser(dest) + + lines = ('{}{} {} {}' + .format(int(x.atime), '-r' * x.redirect, x.url, x.title) + for x in self.select(sort_by='atime', sort_order='asc')) + + with open(dest, 'w', encoding='utf-8') as f: + try: + f.write('\n'.join(lines)) + except OSError as e: + message.error('Could not write history: {}'.format(e)) + else: + message.info("Dumped history to {}.".format(dest)) + def init(parent=None): """Initialize the web history. diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 21af483c9..5c49767e8 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -200,8 +200,14 @@ class SqlTable(QObject): run_query("DELETE FROM {}".format(self._name)) self.changed.emit() - def select(self, sort_by, sort_order, limit): - """Remove all row from the table.""" + def select(self, sort_by, sort_order, limit=-1): + """Remove all row from the table. + + Args: + sort_by: name of column to sort by. + sort_order: 'asc' or 'desc'. + limit: max number of rows in result, defaults to -1 (unlimited). + """ result = run_query('SELECT * FROM {} ORDER BY {} {} LIMIT {}' .format(self._name, sort_by, sort_order, limit)) while result.next(): diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 6efd9ad0e..8174930f0 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -11,49 +11,45 @@ Feature: Page history Scenario: Simple history saving When I open data/numbers/1.txt And I open data/numbers/2.txt - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/numbers/1.txt http://localhost:(port)/data/numbers/2.txt Scenario: History item with title When I open data/title.html - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/title.html Test title Scenario: History item with redirect When I open redirect-to?url=data/title.html without waiting And I wait until data/title.html is loaded - Then the history file should contain: + Then the history should contain: r http://localhost:(port)/redirect-to?url=data/title.html Test title http://localhost:(port)/data/title.html Test title Scenario: History item with spaces in URL When I open data/title with spaces.html - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/title%20with%20spaces.html Test title Scenario: History item with umlauts When I open data/äöü.html - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/%C3%A4%C3%B6%C3%BC.html Chäschüechli - # The following two tests use qute://history instead of checking the - # history file due to a race condition with sqlite. - # https://github.com/qutebrowser/qutebrowser/pull/2295#issuecomment-292786138 @flaky @qtwebengine_todo: Error page message is not implemented Scenario: History with an error When I run :open file:///does/not/exist And I wait for "Error while loading file:///does/not/exist: Error opening /does/not/exist: *" in the log - And I open qute://history/data - Then the page should contain the plaintext "Error loading page: file:///does/not/exist" + Then the history should contain: + file:///does/not/exist Error loading page: file:///does/not/exist @qtwebengine_todo: Error page message is not implemented Scenario: History with a 404 When I open status/404 without waiting And I wait for "Error while loading http://localhost:*/status/404: NOT FOUND" in the log - And I open qute://history/data - Then the page should contain the plaintext "Error loading page: http://localhost:" - And the page should contain the plaintext "/status/404" + Then the history should contain: + http://localhost:(port)/status/404 Error loading page: http://localhost:(port)/status/404 Scenario: History with invalid URL When I run :tab-only @@ -78,19 +74,19 @@ Feature: Page history Scenario: Clearing history When I open data/title.html And I run :history-clear --force - Then the history file should be empty + Then the history should be empty Scenario: Clearing history with confirmation When I open data/title.html And I run :history-clear And I wait for "Asking question <* title='Clear all browsing history?'>, *" in the log And I run :prompt-accept yes - Then the history file should be empty + Then the history should be empty Scenario: History with yanked URL and 'add to history' flag When I open data/hints/html/simple.html And I hint with args "--add-history links yank" and follow a - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/hints/html/simple.html Simple link http://localhost:(port)/data/hello.txt diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 70f23f86b..197bc3d20 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -17,34 +17,33 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . +import logging import os.path +import re +import tempfile import pytest_bdd as bdd -from PyQt5.QtSql import QSqlDatabase - bdd.scenarios('history.feature') -@bdd.then(bdd.parsers.parse("the history file should contain:\n{expected}")) -def check_history(quteproc, httpbin, expected): - path = os.path.join(quteproc.basedir, 'data', 'history.sqlite') - db = QSqlDatabase.addDatabase('QSQLITE') - db.setDatabaseName(path) - assert db.open(), 'Failed to open history database' - query = db.exec_('select * from History') - actual = [] - while query.next(): - rec = query.record() - url = rec.value(0) - title = rec.value(1) - redirect = rec.value(3) - actual.append('{} {} {}'.format('r' * redirect, url, title).strip()) - db = None - QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) - assert actual == expected.replace('(port)', str(httpbin.port)).splitlines() +@bdd.then(bdd.parsers.parse("the history should contain:\n{expected}")) +def check_history(quteproc, expected, httpbin): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, 'history') + quteproc.send_cmd(':debug-dump-history "{}"'.format(path)) + quteproc.wait_for(category='message', loglevel=logging.INFO, + message='Dumped history to {}.'.format(path)) + + with open(path, 'r', encoding='utf-8') as f: + # ignore access times, they will differ in each run + actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() + for line in f.read().splitlines()) + + expected = expected.replace('(port)', str(httpbin.port)) + assert actual == expected -@bdd.then("the history file should be empty") +@bdd.then("the history should be empty") def check_history_empty(quteproc, httpbin): - check_history(quteproc, httpbin, '') + check_history(quteproc, '', httpbin) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index e6bbfcf69..16b86ca4a 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -261,3 +261,18 @@ def test_read_invalid(hist, tmpdir, line): with pytest.raises(Exception): hist.read(str(histfile)) + + +def test_debug_dump_history(hist, tmpdir): + hist.add_url(QUrl('http://example.com/1'), title="Title1", atime=12345) + hist.add_url(QUrl('http://example.com/2'), title="Title2", atime=12346) + hist.add_url(QUrl('http://example.com/3'), title="Title3", atime=12347) + hist.add_url(QUrl('http://example.com/4'), title="Title4", atime=12348, + redirect=True) + histfile = tmpdir / 'history' + hist.debug_dump_history(str(histfile)) + expected = ['12345 http://example.com/1 Title1', + '12346 http://example.com/2 Title2', + '12347 http://example.com/3 Title3', + '12348-r http://example.com/4 Title4'] + assert histfile.read() == '\n'.join(expected) diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index edc156a55..ca736b0ad 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -53,7 +53,8 @@ def test_iter(): @pytest.mark.parametrize('rows, sort_by, sort_order, limit, result', [ ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', 5, [(1, 6), (2, 5), (3, 4)]), ([[2, 5], [1, 6], [3, 4]], 'a', 'desc', 3, [(3, 4), (2, 5), (1, 6)]), - ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]) + ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]), + ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', -1, [(1, 6), (2, 5), (3, 4)]), ]) def test_select(rows, sort_by, sort_order, limit, result): table = sql.SqlTable('Foo', ['a', 'b']) From e67da51662071a63c8dffa1916144944a41def20 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 23 May 2017 09:01:20 -0400 Subject: [PATCH 080/161] Use prepared SQL queries. --- qutebrowser/browser/history.py | 37 +++--- qutebrowser/completion/models/sqlcategory.py | 54 ++++---- qutebrowser/completion/models/urlmodel.py | 1 + qutebrowser/misc/sql.py | 132 +++++++++---------- tests/unit/completion/test_sqlcategory.py | 12 +- 5 files changed, 115 insertions(+), 121 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e19463e75..d8b1f81d5 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -78,6 +78,19 @@ class WebHistory(sql.SqlTable): def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) + self._between_query = sql.Query('SELECT * FROM History ' + 'where not redirect ' + 'and not url like "qute://%" ' + 'and atime > ? ' + 'and atime <= ? ' + 'ORDER BY atime desc') + + self._before_query = sql.Query('SELECT * FROM History ' + 'where not redirect ' + 'and not url like "qute://%" ' + 'and atime <= ? ' + 'ORDER BY atime desc ' + 'limit ? offset ?') def __repr__(self): return utils.get_repr(self, length=len(self)) @@ -101,16 +114,8 @@ class WebHistory(sql.SqlTable): earliest: Omit timestamps earlier than this. latest: Omit timestamps later than this. """ - result = sql.run_query('SELECT * FROM History ' - 'where not redirect ' - 'and not url like "qute://%" ' - 'and atime > {} ' - 'and atime <= {} ' - 'ORDER BY atime desc' - .format(earliest, latest)) - while result.next(): - rec = result.record() - yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + self._between_query.run([earliest, latest]) + return iter(self._between_query) def entries_before(self, latest, limit, offset): """Iterate non-redirect, non-qute entries occurring before a timestamp. @@ -120,16 +125,8 @@ class WebHistory(sql.SqlTable): limit: Max number of entries to include. offset: Number of entries to skip. """ - result = sql.run_query('SELECT * FROM History ' - 'where not redirect ' - 'and not url like "qute://%" ' - 'and atime <= {} ' - 'ORDER BY atime desc ' - 'limit {} offset {}' - .format(latest, limit, offset)) - while result.next(): - rec = result.record() - yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + self._before_query.run([latest, limit, offset]) + return iter(self._before_query) @cmdutils.register(name='history-clear', instance='web-history') def clear(self, force=False): diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 9f78163ca..543ea2ade 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -29,12 +29,13 @@ from qutebrowser.misc import sql class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" - def __init__(self, name, *, sort_by=None, sort_order=None, select='*', - where=None, group_by=None, parent=None): + def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, + select='*', where=None, group_by=None, parent=None): """Create a new completion category backed by a sql table. Args: name: Name of category, and the table in the database. + filter_fields: Names of fields to apply filter pattern to. select: A custom result column expression for the select statement. where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. @@ -42,11 +43,24 @@ class SqlCategory(QSqlQueryModel): """ super().__init__(parent=parent) self.name = name - self._sort_by = sort_by - self._sort_order = sort_order - self._select = select - self._where = where - self._group_by = group_by + + querystr = 'select {} from {} where ('.format(select, name) + # the incoming pattern will have literal % and _ escaped with '\' + # we need to tell sql to treat '\' as an escape character + querystr += ' or '.join("{} like ? escape '\\'".format(f) + for f in filter_fields) + querystr += ')' + + if where: + querystr += ' and ' + where + if group_by: + querystr += ' group by {}'.format(group_by) + if sort_by: + assert sort_order in ['asc', 'desc'] + querystr += ' order by {} {}'.format(sort_by, sort_order) + + self._query = sql.Query(querystr) + self._param_count=len(filter_fields) self.set_pattern('', [0]) def set_pattern(self, pattern, columns_to_filter): @@ -56,31 +70,13 @@ class SqlCategory(QSqlQueryModel): pattern: string pattern to filter by. columns_to_filter: indices of columns to apply pattern to. """ - query = sql.run_query('select * from {} limit 1'.format(self.name)) - fields = [query.record().fieldName(i) for i in columns_to_filter] - - querystr = 'select {} from {} where ('.format(self._select, self.name) - # the incoming pattern will have literal % and _ escaped with '\' - # we need to tell sql to treat '\' as an escape character - querystr += ' or '.join("{} like ? escape '\\'".format(f) - for f in fields) - querystr += ')' - if self._where: - querystr += ' and ' + self._where - - if self._group_by: - querystr += ' group by {}'.format(self._group_by) - - if self._sort_by: - assert self._sort_order in ['asc', 'desc'] - querystr += ' order by {} {}'.format(self._sort_by, - self._sort_order) - + # TODO: eliminate columns_to_filter + #assert len(columns_to_filter) == self._param_count # escape to treat a user input % or _ as a literal, not a wildcard pattern = pattern.replace('%', '\\%') pattern = pattern.replace('_', '\\_') # treat spaces as wildcards to match any of the typed words pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) - query = sql.run_query(querystr, [pattern for _ in fields]) - self.setQuery(query) + self._query.run([pattern] * self._param_count) + self.setQuery(self._query) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 9e601cd39..29ecdca9f 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -77,6 +77,7 @@ def url(): select_time = "strftime('{}', max(atime), 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( 'History', sort_order='desc', sort_by='atime', group_by='url', + filter_fields=['url', 'title'], select='url, title, {}'.format(select_time), where='not redirect') model.add_category(hist_cat) return model diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 5c49767e8..5acc620fb 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -50,56 +50,45 @@ def close(): def version(): """Return the sqlite version string.""" - result = run_query("select sqlite_version()") - result.next() - return result.record().value(0) + q = Query("select sqlite_version()") + q.run() + return q.value() -def _prepare_query(querystr): - log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) - database = QSqlDatabase.database() - query = QSqlQuery(database) - query.prepare(querystr) - return query +class Query(QSqlQuery): + """A prepared SQL Query.""" -def run_query(querystr, values=None): - """Run the given SQL query string on the database. + def __init__(self, querystr): + super().__init__(QSqlDatabase.database()) + log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) + self.prepare(querystr) - Args: - values: A list of positional parameter bindings. - """ - query = _prepare_query(querystr) - for val in values or []: - query.addBindValue(val) - log.sql.debug('Query bindings: {}'.format(query.boundValues())) - if not query.exec_(): - raise SqlException('Failed to exec query "{}": "{}"'.format( - querystr, query.lastError().text())) - return query + def __iter__(self): + assert self.isActive(), "Cannot iterate inactive query" + rec = self.record() + fields = [rec.fieldName(i) for i in range(rec.count())] + rowtype = collections.namedtuple('ResultRow', fields) + while self.next(): + rec = self.record() + yield rowtype(*[rec.value(i) for i in range(rec.count())]) -def run_batch(querystr, values): - """Run the given SQL query string on the database in batch mode. + def run(self, values=None): + """Execute the prepared query.""" + log.sql.debug('Running SQL query: "{}"'.format(self.lastQuery())) + for val in values or []: + self.addBindValue(val) + log.sql.debug('self bindings: {}'.format(self.boundValues())) + if not self.exec_(): + raise SqlException('Failed to exec self "{}": "{}"'.format( + self.lastself(), self.lastError().text())) - Args: - values: A list of lists, where each inner list contains positional - bindings for one run of the batch. - """ - query = _prepare_query(querystr) - transposed = [list(row) for row in zip(*values)] - for val in transposed: - query.addBindValue(val) - log.sql.debug('Batch Query bindings: {}'.format(query.boundValues())) - - db = QSqlDatabase.database() - db.transaction() - if not query.execBatch(): - raise SqlException('Failed to exec query "{}": "{}"'.format( - querystr, query.lastError().text())) - db.commit() - - return query + def value(self): + """Return the result of a single-value query (e.g. an EXISTS).""" + ok = self.next() + assert ok, "No result for single-result query" + return self.record().value(0) class SqlTable(QObject): @@ -127,17 +116,17 @@ class SqlTable(QObject): """ super().__init__(parent) self._name = name - run_query("CREATE TABLE IF NOT EXISTS {} ({})" + q = Query("CREATE TABLE IF NOT EXISTS {} ({})" .format(name, ','.join(fields))) + q.run() # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) def __iter__(self): """Iterate rows in the table.""" - result = run_query("SELECT * FROM {}".format(self._name)) - while result.next(): - rec = result.record() - yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + q = Query("SELECT * FROM {}".format(self._name)) + q.run() + return iter(q) def contains(self, field, value): """Return whether the table contains the matching item. @@ -146,16 +135,16 @@ class SqlTable(QObject): field: Field to match. value: Value to check for the given field. """ - query = run_query("Select EXISTS(SELECT * FROM {} where {} = ?)" - .format(self._name, field), [value]) - query.next() - return query.value(0) + q = Query("Select EXISTS(SELECT * FROM {} where {} = ?)" + .format(self._name, field)) + q.run([value]) + return q.value() def __len__(self): """Return the count of rows in the table.""" - result = run_query("SELECT count(*) FROM {}".format(self._name)) - result.next() - return result.value(0) + q = Query("SELECT count(*) FROM {}".format(self._name)) + q.run() + return q.value() def delete(self, value, field): """Remove all rows for which `field` equals `value`. @@ -167,9 +156,9 @@ class SqlTable(QObject): Return: The number of rows deleted. """ - query = run_query("DELETE FROM {} where {} = ?".format( - self._name, field), [value]) - if not query.numRowsAffected(): + q = Query("DELETE FROM {} where {} = ?".format(self._name, field)) + q.run([value]) + if not q.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() @@ -180,8 +169,8 @@ class SqlTable(QObject): values: A list of values to insert. """ paramstr = ','.join(['?'] * len(values)) - run_query("INSERT INTO {} values({})".format(self._name, paramstr), - values) + q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q.run(values) self.changed.emit() def insert_batch(self, rows): @@ -191,13 +180,23 @@ class SqlTable(QObject): rows: A list of lists, where each sub-list is a row. """ paramstr = ','.join(['?'] * len(rows[0])) - run_batch("INSERT INTO {} values({})".format(self._name, paramstr), - rows) + q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + + transposed = [list(row) for row in zip(*rows)] + for val in transposed: + q.addBindValue(val) + + db = QSqlDatabase.database() + db.transaction() + if not q.execBatch(): + raise SqlException('Failed to exec query "{}": "{}"'.format( + q.lastQuery(), q.lastError().text())) + db.commit() self.changed.emit() def delete_all(self): """Remove all row from the table.""" - run_query("DELETE FROM {}".format(self._name)) + Query("DELETE FROM {}".format(self._name)).run() self.changed.emit() def select(self, sort_by, sort_order, limit=-1): @@ -208,8 +207,7 @@ class SqlTable(QObject): sort_order: 'asc' or 'desc'. limit: max number of rows in result, defaults to -1 (unlimited). """ - result = run_query('SELECT * FROM {} ORDER BY {} {} LIMIT {}' - .format(self._name, sort_by, sort_order, limit)) - while result.next(): - rec = result.record() - yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + q = Query('SELECT * FROM {} ORDER BY {} {} LIMIT ?' + .format(self._name, sort_by, sort_order)) + q.run([limit]) + return q diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index f29589717..3d0b07d07 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -74,7 +74,7 @@ def test_sorting(sort_by, sort_order, data, expected): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: table.insert(row) - cat = sqlcategory.SqlCategory('Foo', sort_by=sort_by, + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) _validate(cat, expected) @@ -129,7 +129,8 @@ def test_set_pattern(pattern, filter_cols, before, after): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: table.insert(row) - cat = sqlcategory.SqlCategory('Foo') + filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] + cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) cat.set_pattern(pattern, filter_cols) _validate(cat, after) @@ -137,7 +138,7 @@ def test_set_pattern(pattern, filter_cols, before, after): def test_select(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert(['foo', 'bar', 'baz']) - cat = sqlcategory.SqlCategory('Foo', select='b, c, a') + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') _validate(cat, [('bar', 'baz', 'foo')]) @@ -145,7 +146,7 @@ def test_where(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert(['foo', 'bar', False]) table.insert(['baz', 'biz', True]) - cat = sqlcategory.SqlCategory('Foo', where='not c') + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') _validate(cat, [('foo', 'bar', False)]) @@ -155,7 +156,8 @@ def test_group(): table.insert(['bar', 3]) table.insert(['foo', 2]) table.insert(['bar', 0]) - cat = sqlcategory.SqlCategory('Foo', select='a, max(b)', group_by='a') + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], + select='a, max(b)', group_by='a') _validate(cat, [('bar', 3), ('foo', 2)]) From b1b521e0c2ad814bf3eda76b69c47560bf027c12 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 23 May 2017 22:20:25 -0400 Subject: [PATCH 081/161] Fix two history tests added recently. These were added on master and needed to be adjusted slightly for the new history check (which doesn't rely on reading a history file anymore). --- tests/end2end/features/history.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 8174930f0..9317bde64 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -61,14 +61,14 @@ Feature: Page history When I open data/data_link.html And I run :click-element id link And I wait until data:;base64,cXV0ZWJyb3dzZXI= is loaded - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/data_link.html data: link Scenario: History with view-source URL When I open data/title.html And I run :view-source And I wait for "Changing title for idx * to 'Source for http://localhost:*/data/title.html'" in the log - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/title.html Test title Scenario: Clearing history From 39b561a1821689a933ca31a64b2ece47e03adff5 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 24 May 2017 07:38:24 -0400 Subject: [PATCH 082/161] Fix BaseLineParser::test_double_open. Don't tie the test to a particular error message. Ths failed because a typo was fixed (AppendLineParser -> LineParser). --- tests/unit/misc/test_lineparser.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/misc/test_lineparser.py b/tests/unit/misc/test_lineparser.py index adae2485d..dcdb724cd 100644 --- a/tests/unit/misc/test_lineparser.py +++ b/tests/unit/misc/test_lineparser.py @@ -58,8 +58,7 @@ class TestBaseLineParser: mocker.patch('builtins.open', mock.mock_open()) with lineparser._open('r'): - with pytest.raises(IOError, match="Refusing to double-open " - "AppendLineParser."): + with pytest.raises(IOError): with lineparser._open('r'): pass From 2c501f7fb712289656da5895262d4bf1338b21d4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 24 May 2017 12:44:57 -0400 Subject: [PATCH 083/161] Fix url completion benchmark. Still had old code from pre-SQL era. --- tests/unit/completion/test_models.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index cea9d0ccd..8666ecc5b 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -580,21 +580,21 @@ def test_url_completion_benchmark(benchmark, config_stub, config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 1000} - entries = [history.Entry( + entries = [web_history_stub.Entry( atime=i, - url=QUrl('http://example.com/{}'.format(i)), - title='title{}'.format(i)) + url='http://example.com/{}'.format(i), + title='title{}'.format(i), + redirect=False) for i in range(100000)] - web_history_stub.history_dict = collections.OrderedDict( - ((e.url_str(), e) for e in entries)) + web_history_stub.insert_batch(entries) quickmark_manager_stub.marks = collections.OrderedDict( - (e.title, e.url_str()) + (e.title, e.url) for e in entries[0:1000]) bookmark_manager_stub.marks = collections.OrderedDict( - (e.url_str(), e.title) + (e.url, e.title) for e in entries[0:1000]) def bench(): From cf89ffa9710afacb0b928dadda74774ea42f3ce6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 25 May 2017 07:27:14 -0400 Subject: [PATCH 084/161] Fix pylint/flake8 errors --- qutebrowser/completion/models/sqlcategory.py | 4 ++-- qutebrowser/misc/sql.py | 4 ++-- tests/unit/completion/test_completionmodel.py | 2 +- tests/unit/completion/test_models.py | 1 - tests/unit/utils/test_version.py | 1 + 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 543ea2ade..5b8cf4482 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -60,10 +60,10 @@ class SqlCategory(QSqlQueryModel): querystr += ' order by {} {}'.format(sort_by, sort_order) self._query = sql.Query(querystr) - self._param_count=len(filter_fields) + self._param_count = len(filter_fields) self.set_pattern('', [0]) - def set_pattern(self, pattern, columns_to_filter): + def set_pattern(self, pattern, _columns_to_filter): """Set the pattern used to filter results. Args: diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 5acc620fb..0b81fc0f9 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -19,13 +19,13 @@ """Provides access to an in-memory sqlite database.""" +import collections + from PyQt5.QtCore import QObject, pyqtSignal from PyQt5.QtSql import QSqlDatabase, QSqlQuery from qutebrowser.utils import log -import collections - class SqlException(Exception): diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 05af902a3..d3cd23f68 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -19,8 +19,8 @@ """Tests for CompletionModel.""" -import hypothesis from unittest import mock +import hypothesis from hypothesis import strategies from qutebrowser.completion.models import completionmodel diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 8666ecc5b..c46287926 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -26,7 +26,6 @@ import pytest from PyQt5.QtCore import QUrl from PyQt5.QtWidgets import QTreeView -from qutebrowser.browser import history from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value from qutebrowser.misc import sql diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index c0c731f88..11bf326fb 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -798,6 +798,7 @@ def test_chromium_version_unpatched(qapp): assert version._chromium_version() not in ['', 'unknown', 'unavailable'] +# pylint: disable=too-many-locals @pytest.mark.parametrize(['git_commit', 'frozen', 'style', 'with_webkit', 'known_distribution'], [ (True, False, True, True, True), # normal From 8fb6f45bec9c82d26a653a1e96c6701fad12b020 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 25 May 2017 12:59:05 -0400 Subject: [PATCH 085/161] Don't set pattern in SqlCategory constructor. This will be called by the Completer after construction anyways, this was a duplicate call that could be expensive. --- qutebrowser/completion/models/sqlcategory.py | 3 +-- tests/unit/completion/test_sqlcategory.py | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 5b8cf4482..56d7ec399 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -61,9 +61,8 @@ class SqlCategory(QSqlQueryModel): self._query = sql.Query(querystr) self._param_count = len(filter_fields) - self.set_pattern('', [0]) - def set_pattern(self, pattern, _columns_to_filter): + def set_pattern(self, pattern, _columns_to_filter=None): """Set the pattern used to filter results. Args: diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 3d0b07d07..2f3b8474d 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -76,6 +76,7 @@ def test_sorting(sort_by, sort_order, data, expected): table.insert(row) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) + cat.set_pattern('') _validate(cat, expected) @@ -139,6 +140,7 @@ def test_select(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert(['foo', 'bar', 'baz']) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') + cat.set_pattern('') _validate(cat, [('bar', 'baz', 'foo')]) @@ -147,6 +149,7 @@ def test_where(): table.insert(['foo', 'bar', False]) table.insert(['baz', 'biz', True]) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') + cat.set_pattern('') _validate(cat, [('foo', 'bar', False)]) @@ -158,6 +161,7 @@ def test_group(): table.insert(['bar', 0]) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='a, max(b)', group_by='a') + cat.set_pattern('') _validate(cat, [('bar', 3), ('foo', 2)]) From 2cd02be7b108550b204095512be03d5eaff148c8 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 29 May 2017 22:58:11 -0400 Subject: [PATCH 086/161] Remove CompletionModel.columns_to_filter. Instead set this on inidividual categories, as that is where it actually gets used. This makes it easier for SqlCompletionCategory to reuse a prepared query (as it gets the filter field names in its constructor). --- qutebrowser/completion/completiondelegate.py | 2 +- .../completion/models/completionmodel.py | 19 +++++++++++++------ qutebrowser/completion/models/listcategory.py | 7 +++---- qutebrowser/completion/models/miscmodels.py | 7 ++++--- qutebrowser/completion/models/sqlcategory.py | 5 ++++- qutebrowser/completion/models/urlmodel.py | 7 ++++--- tests/unit/completion/test_completionmodel.py | 5 ++--- tests/unit/completion/test_listcategory.py | 5 +++-- tests/unit/completion/test_sqlcategory.py | 2 +- 9 files changed, 35 insertions(+), 24 deletions(-) diff --git a/qutebrowser/completion/completiondelegate.py b/qutebrowser/completion/completiondelegate.py index 776b2164c..5813d6dcb 100644 --- a/qutebrowser/completion/completiondelegate.py +++ b/qutebrowser/completion/completiondelegate.py @@ -197,7 +197,7 @@ class CompletionItemDelegate(QStyledItemDelegate): if index.parent().isValid(): pattern = index.model().pattern - columns_to_filter = index.model().columns_to_filter + columns_to_filter = index.model().columns_to_filter(index) if index.column() in columns_to_filter and pattern: repl = r'\g<0>' text = re.sub(re.escape(pattern).replace(r'\ ', r'|'), diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 0ee81fdf5..9543792ed 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -34,15 +34,13 @@ class CompletionModel(QAbstractItemModel): Attributes: column_widths: The width percentages of the columns used in the completion view. - columns_to_filter: A list of indices of columns to apply the filter to. pattern: Current filter pattern, used for highlighting. _categories: The sub-categories. """ - def __init__(self, *, column_widths=(30, 70, 0), columns_to_filter=None, + def __init__(self, *, column_widths=(30, 70, 0), delete_cur_item=None, parent=None): super().__init__(parent) - self.columns_to_filter = columns_to_filter or [0] self.column_widths = column_widths self._categories = [] self.pattern = '' @@ -173,8 +171,6 @@ class CompletionModel(QAbstractItemModel): def set_pattern(self, pattern): """Set the filter pattern for all categories. - This will apply to the fields indicated in columns_to_filter. - Args: pattern: The filter pattern to set. """ @@ -182,7 +178,7 @@ class CompletionModel(QAbstractItemModel): # TODO: should pattern be saved in the view layer instead? self.pattern = pattern for cat in self._categories: - cat.set_pattern(pattern, self.columns_to_filter) + cat.set_pattern(pattern) def first_item(self): """Return the index of the first child (non-category) in the model.""" @@ -204,3 +200,14 @@ class CompletionModel(QAbstractItemModel): qtutils.ensure_valid(index) return index return QModelIndex() + + def columns_to_filter(self, index): + """Return the column indices the filter pattern applies to. + + Args: + index: index of the item to check. + + Return: A list of integers. + """ + cat = self._cat_from_idx(index.parent()) + return cat.columns_to_filter if cat else [] diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index d24fd74d0..fbad3b063 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -35,25 +35,24 @@ class ListCategory(QSortFilterProxyModel): """Expose a list of items as a category for the CompletionModel.""" - def __init__(self, name, items, parent=None): + def __init__(self, name, items, columns_to_filter=None, parent=None): super().__init__(parent) self.name = name self.srcmodel = QStandardItemModel(parent=self) self.pattern = '' self.pattern_re = None - self.columns_to_filter = None + self.columns_to_filter = columns_to_filter or [0] for item in items: self.srcmodel.appendRow([QStandardItem(x) for x in item]) self.setSourceModel(self.srcmodel) - def set_pattern(self, val, columns_to_filter): + def set_pattern(self, val): """Setter for pattern. Args: val: The value to set. """ with debug.log_time(log.completion, 'Setting filter pattern'): - self.columns_to_filter = columns_to_filter self.pattern = val val = re.sub(r' +', r' ', val) # See #1919 val = re.escape(val) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 368e1e6c6..62818f767 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -110,8 +110,7 @@ def buffer(): model = completionmodel.CompletionModel( column_widths=(6, 40, 54), - delete_cur_item=delete_buffer, - columns_to_filter=[idx_column, url_column, text_column]) + delete_cur_item=delete_buffer) for win_id in objreg.window_registry: tabbed_browser = objreg.get('tabbed-browser', scope='window', @@ -124,8 +123,10 @@ def buffer(): tabs.append(("{}/{}".format(win_id, idx + 1), tab.url().toDisplayString(), tabbed_browser.page_title(idx))) - cat = listcategory.ListCategory("{}".format(win_id), tabs) + cat = listcategory.ListCategory("{}".format(win_id), tabs, + columns_to_filter=[idx_column, url_column, text_column]) model.add_category(cat) + return model diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 56d7ec399..b8cf07ffa 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -61,8 +61,11 @@ class SqlCategory(QSqlQueryModel): self._query = sql.Query(querystr) self._param_count = len(filter_fields) + rec = self._query.record() + # will this work? + self.columns_to_filter = [rec.indexOf(n) for n in filter_fields] - def set_pattern(self, pattern, _columns_to_filter=None): + def set_pattern(self, pattern): """Set the pattern used to filter results. Args: diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 29ecdca9f..9e2571848 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -63,15 +63,16 @@ def url(): """ model = completionmodel.CompletionModel( column_widths=(40, 50, 10), - columns_to_filter=[_URLCOL, _TEXTCOL], delete_cur_item=_delete_url) quickmarks = ((url, name) for (name, url) in objreg.get('quickmark-manager').marks.items()) bookmarks = objreg.get('bookmark-manager').marks.items() - model.add_category(listcategory.ListCategory('Quickmarks', quickmarks)) - model.add_category(listcategory.ListCategory('Bookmarks', bookmarks)) + model.add_category(listcategory.ListCategory('Quickmarks', quickmarks, + columns_to_filter=[0, 1])) + model.add_category(listcategory.ListCategory('Bookmarks', bookmarks, + columns_to_filter=[0, 1])) timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', max(atime), 'unixepoch')".format(timefmt) diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index d3cd23f68..8d43f2883 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -65,11 +65,10 @@ def test_count(counts): @hypothesis.given(strategies.text()) def test_set_pattern(pat): """Validate the filtering and sorting results of set_pattern.""" - cols = [1, 2, 3] - model = completionmodel.CompletionModel(columns_to_filter=cols) + model = completionmodel.CompletionModel() cats = [mock.Mock(spec=['set_pattern'])] * 3 for c in cats: c.set_pattern = mock.Mock() model.add_category(c) model.set_pattern(pat) - assert all(c.set_pattern.called_with([pat, cols]) for c in cats) + assert all(c.set_pattern.called_with([pat]) for c in cats) diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index 0b4426bdb..a0c300473 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -65,6 +65,7 @@ def _validate(cat, expected): ]) def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" - cat = listcategory.ListCategory('Foo', before) - cat.set_pattern(pattern, filter_cols) + cat = listcategory.ListCategory('Foo', before, + columns_to_filter=filter_cols) + cat.set_pattern(pattern) _validate(cat, after) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 2f3b8474d..1b4190d21 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -132,7 +132,7 @@ def test_set_pattern(pattern, filter_cols, before, after): table.insert(row) filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) - cat.set_pattern(pattern, filter_cols) + cat.set_pattern(pattern) _validate(cat, after) From c297f047d224d9d681a45b5712bcb67a80044b55 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 31 May 2017 12:35:43 -0400 Subject: [PATCH 087/161] Don't regenerate completion model needlessly. If the completion model would stay the same, just keep it and update the filter pattern rather than instantiating a new model each time the pattern changes. --- qutebrowser/completion/completer.py | 37 ++++++++++++------- qutebrowser/completion/completionwidget.py | 28 ++++++-------- qutebrowser/completion/models/miscmodels.py | 1 + tests/unit/completion/test_completer.py | 9 ++--- .../unit/completion/test_completionwidget.py | 3 +- 5 files changed, 43 insertions(+), 35 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 94c7f9352..b70325abe 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -39,6 +39,7 @@ class Completer(QObject): _last_cursor_pos: The old cursor position so we avoid double completion updates. _last_text: The old command text so we avoid double completion updates. + _last_completion_func: The completion function used for the last text. """ def __init__(self, cmd, win_id, parent=None): @@ -52,6 +53,7 @@ class Completer(QObject): self._timer.timeout.connect(self._update_completion) self._last_cursor_pos = None self._last_text = None + self._last_completion_func = None self._cmd.update_completion.connect(self.schedule_completion_update) def __repr__(self): @@ -63,7 +65,7 @@ class Completer(QObject): return completion.model() def _get_new_completion(self, before_cursor, under_cursor): - """Get a new completion. + """Get the completion function based on the current command text. Args: before_cursor: The command chunks before the cursor. @@ -81,7 +83,7 @@ class Completer(QObject): if not before_cursor: # '|' or 'set|' log.completion.debug('Starting command completion') - return miscmodels.command() + return miscmodels.command try: cmd = cmdutils.cmd_dict[before_cursor[0]] except KeyError: @@ -90,17 +92,11 @@ class Completer(QObject): return None argpos = len(before_cursor) - 1 try: - completion = cmd.get_pos_arg_info(argpos).completion + func = cmd.get_pos_arg_info(argpos).completion except IndexError: log.completion.debug("No completion in position {}".format(argpos)) return None - if completion is None: - return None - - model = completion(*before_cursor[1:]) - log.completion.debug('Starting {} completion' - .format(completion.__name__)) - return model + return func def _quote(self, s): """Quote s if it needs quoting for the commandline. @@ -223,9 +219,24 @@ class Completer(QObject): before_cursor, pattern, after_cursor)) pattern = pattern.strip("'\"") - model = self._get_new_completion(before_cursor, pattern) - log.completion.debug("Setting pattern to '{}'".format(pattern)) - completion.set_model(model, pattern) + func = self._get_new_completion(before_cursor, pattern) + + if func is None: + log.completion.debug('Clearing completion') + completion.set_model(None) + elif func != self._last_completion_func: + self._last_completion_func = func + args = (x for x in before_cursor[1:] if not x.startswith('-')) + model = func(*args) + log.completion.debug('Starting {} completion' + .format(func.__name__)) + completion.set_model(model) + completion.set_pattern(pattern) + else: + log.completion.debug('Setting pattern {}'.format(pattern)) + completion.set_pattern(pattern) + + self._last_completion_func = None def _change_completed_part(self, newtext, before, after, immediate=False): """Change the part we're currently completing in the commandline. diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 5e207d68d..a476e70dc 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -261,32 +261,29 @@ class CompletionView(QTreeView): elif config.get('completion', 'show') == 'auto': self.show() - def set_model(self, model, pattern=None): + def set_model(self, model): """Switch completion to a new model. Called from on_update_completion(). Args: model: The model to use. - pattern: The filter pattern to set (what the user entered). """ if model is None: self._active = False self.hide() return + if self.model() is not None: + self.model().deleteLater() + self.selectionModel().deleteLater() + model.setParent(self) - old_model = self.model() - if model is not old_model: - sel_model = self.selectionModel() + self.setModel(model) + self._column_widths = model.column_widths + self._active = True - self.setModel(model) - self._active = True - - if sel_model is not None: - sel_model.deleteLater() - if old_model is not None: - old_model.deleteLater() + self.set_pattern('') if (config.get('completion', 'show') == 'always' and model.count() > 0): @@ -297,12 +294,11 @@ class CompletionView(QTreeView): for i in range(model.rowCount()): self.expand(model.index(i, 0)) - if pattern is not None: - model.set_pattern(pattern) - - self._column_widths = model.column_widths + def set_pattern(self, pattern): + self.model().set_pattern(pattern) self._resize_columns() self._maybe_update_geometry() + self.show() def _maybe_update_geometry(self): """Emit the update_geometry signal if the config says so.""" diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 62818f767..233ee8dcc 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -137,6 +137,7 @@ def bind(_key): _key: the key being bound. """ # TODO: offer a 'Current binding' completion based on the key. + print(_key) model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) model.add_category(listcategory.ListCategory("Commands", cmdlist)) diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index c407aeab8..537baad46 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -191,15 +191,14 @@ def test_update_completion(txt, kind, pattern, pos_args, status_command_stub, # this test uses | as a placeholder for the current cursor position _set_cmd_prompt(status_command_stub, txt) completer_obj.schedule_completion_update() - assert completion_widget_stub.set_model.call_count == 1 - args = completion_widget_stub.set_model.call_args[0] if kind is None: - assert args[0] is None + assert completion_widget_stub.set_pattern.call_count == 0 else: - model = args[0] + assert completion_widget_stub.set_model.call_count == 1 + model = completion_widget_stub.set_model.call_args[0][0] assert model.kind == kind assert model.pos_args == pos_args - assert args[1] == pattern + completion_widget_stub.set_pattern.assert_called_once_with(pattern) @pytest.mark.parametrize('before, newtxt, after', [ diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 7985bfa5b..66f6e7f89 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -84,7 +84,8 @@ def test_set_model(completionview): def test_set_pattern(completionview): model = completionmodel.CompletionModel() model.set_pattern = mock.Mock() - completionview.set_model(model, 'foo') + completionview.set_model(model) + completionview.set_pattern('foo') model.set_pattern.assert_called_with('foo') From a01c76db54a8a997a1d46d411aa840e5a6ed7436 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 2 Jun 2017 07:47:28 -0400 Subject: [PATCH 088/161] Remove 'group by' from url completion query. This seemed to have a significant performance impact. Removing it means that instead of just seeing the most recent atime for a given url, you will see multiple entries. --- qutebrowser/completion/models/urlmodel.py | 4 ++-- tests/unit/completion/test_models.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 9e2571848..b106da366 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -75,9 +75,9 @@ def url(): columns_to_filter=[0, 1])) timefmt = config.get('completion', 'timestamp-format') - select_time = "strftime('{}', max(atime), 'unixepoch')".format(timefmt) + select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'History', sort_order='desc', sort_by='atime', group_by='url', + 'History', sort_order='desc', sort_by='atime', filter_fields=['url', 'title'], select='url, title, {}'.format(select_time), where='not redirect') model.add_category(hist_cat) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index c46287926..46730cc21 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -306,7 +306,9 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, "History": [ ('https://github.com', 'https://github.com', '2016-05-01'), ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + ('https://python.org', 'Welcome to Python.org', '2016-02-08'), ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ('https://python.org', 'Welcome to Python.org', '2014-03-08'), ], }) From 42243d3d9765ac52c3cf2185552d34224a5805b2 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 5 Jun 2017 07:40:48 -0400 Subject: [PATCH 089/161] Add more performance logging to completion. --- qutebrowser/completion/completer.py | 11 ++++++----- qutebrowser/completion/completionwidget.py | 11 ++++++----- qutebrowser/completion/models/listcategory.py | 17 ++++++++--------- tests/end2end/features/test_completion_bdd.py | 2 +- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index b70325abe..ae2e9e4bb 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -23,7 +23,7 @@ from PyQt5.QtCore import pyqtSlot, QObject, QTimer from qutebrowser.config import config from qutebrowser.commands import cmdutils, runners -from qutebrowser.utils import log, utils +from qutebrowser.utils import log, utils, debug from qutebrowser.completion.models import miscmodels @@ -227,10 +227,11 @@ class Completer(QObject): elif func != self._last_completion_func: self._last_completion_func = func args = (x for x in before_cursor[1:] if not x.startswith('-')) - model = func(*args) - log.completion.debug('Starting {} completion' - .format(func.__name__)) - completion.set_model(model) + with debug.log_time(log.completion, + 'Instantiate {} completion'.format(func.__name__)): + model = func(*args) + with debug.log_time(log.completion, 'Set completion model'): + completion.set_model(model) completion.set_pattern(pattern) else: log.completion.debug('Setting pattern {}'.format(pattern)) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index a476e70dc..1f8b3d9f4 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -28,7 +28,7 @@ from PyQt5.QtCore import pyqtSlot, pyqtSignal, Qt, QItemSelectionModel, QSize from qutebrowser.config import config, style from qutebrowser.completion import completiondelegate -from qutebrowser.utils import utils, usertypes, objreg +from qutebrowser.utils import utils, usertypes, objreg, debug, log from qutebrowser.commands import cmdexc, cmdutils @@ -295,10 +295,11 @@ class CompletionView(QTreeView): self.expand(model.index(i, 0)) def set_pattern(self, pattern): - self.model().set_pattern(pattern) - self._resize_columns() - self._maybe_update_geometry() - self.show() + with debug.log_time(log.completion, 'Set pattern {}'.format(pattern)): + self.model().set_pattern(pattern) + self._resize_columns() + self._maybe_update_geometry() + self.show() def _maybe_update_geometry(self): """Emit the update_geometry signal if the config says so.""" diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index fbad3b063..75cc2642c 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -52,15 +52,14 @@ class ListCategory(QSortFilterProxyModel): Args: val: The value to set. """ - with debug.log_time(log.completion, 'Setting filter pattern'): - self.pattern = val - val = re.sub(r' +', r' ', val) # See #1919 - val = re.escape(val) - val = val.replace(r'\ ', '.*') - self.pattern_re = re.compile(val, re.IGNORECASE) - self.invalidate() - sortcol = 0 - self.sort(sortcol) + self.pattern = val + val = re.sub(r' +', r' ', val) # See #1919 + val = re.escape(val) + val = val.replace(r'\ ', '.*') + self.pattern_re = re.compile(val, re.IGNORECASE) + self.invalidate() + sortcol = 0 + self.sort(sortcol) def filterAcceptsRow(self, row, parent): """Custom filter implementation. diff --git a/tests/end2end/features/test_completion_bdd.py b/tests/end2end/features/test_completion_bdd.py index 4a06bfbc0..82e2df030 100644 --- a/tests/end2end/features/test_completion_bdd.py +++ b/tests/end2end/features/test_completion_bdd.py @@ -24,5 +24,5 @@ bdd.scenarios('completion.feature') @bdd.then(bdd.parsers.parse("the completion model should be {model}")) def check_model(quteproc, model): """Make sure the completion model was set to something.""" - pattern = "Starting {} completion".format(model) + pattern = "Starting {} completion *".format(model) quteproc.wait_for(message=pattern) From 565ba23f8cd33fd777324a85794cd3dc7dc4b72f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 6 Jun 2017 20:40:24 -0400 Subject: [PATCH 090/161] Don't instantiate completion models nedlessly. For real this time. A mistake on the last commit like this meant models were still spuriously instantiated. Now that the completion model is reused, the layoutChanged signal needs to be forwarded through, otherwise the view will not update. --- qutebrowser/completion/completer.py | 13 ++++++------- qutebrowser/completion/completionwidget.py | 2 -- qutebrowser/completion/models/completionmodel.py | 1 + tests/unit/completion/test_completionmodel.py | 4 ++-- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index ae2e9e4bb..491fe2e04 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -224,20 +224,19 @@ class Completer(QObject): if func is None: log.completion.debug('Clearing completion') completion.set_model(None) - elif func != self._last_completion_func: + self._last_completion_func = None + return + + if func != self._last_completion_func: self._last_completion_func = func args = (x for x in before_cursor[1:] if not x.startswith('-')) with debug.log_time(log.completion, - 'Instantiate {} completion'.format(func.__name__)): + 'Starting {} completion'.format(func.__name__)): model = func(*args) with debug.log_time(log.completion, 'Set completion model'): completion.set_model(model) - completion.set_pattern(pattern) - else: - log.completion.debug('Setting pattern {}'.format(pattern)) - completion.set_pattern(pattern) - self._last_completion_func = None + completion.set_pattern(pattern) def _change_completed_part(self, newtext, before, after, immediate=False): """Change the part we're currently completing in the commandline. diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 1f8b3d9f4..711301b4c 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -283,8 +283,6 @@ class CompletionView(QTreeView): self._column_widths = model.column_widths self._active = True - self.set_pattern('') - if (config.get('completion', 'show') == 'always' and model.count() > 0): self.show() diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 9543792ed..cbc20470c 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -63,6 +63,7 @@ class CompletionModel(QAbstractItemModel): def add_category(self, cat): """Add a completion category to the model.""" self._categories.append(cat) + cat.layoutChanged.connect(self.layoutChanged) def data(self, index, role=Qt.DisplayRole): """Return the item data for index. diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 8d43f2883..8f8acced2 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -56,7 +56,7 @@ def test_first_last_item(counts): def test_count(counts): model = completionmodel.CompletionModel() for c in counts: - cat = mock.Mock(spec=['rowCount']) + cat = mock.Mock(spec=['rowCount', 'layoutChanged']) cat.rowCount = mock.Mock(return_value=c) model.add_category(cat) assert model.count() == sum(counts) @@ -66,7 +66,7 @@ def test_count(counts): def test_set_pattern(pat): """Validate the filtering and sorting results of set_pattern.""" model = completionmodel.CompletionModel() - cats = [mock.Mock(spec=['set_pattern'])] * 3 + cats = [mock.Mock(spec=['set_pattern', 'layoutChanged'])] * 3 for c in cats: c.set_pattern = mock.Mock() model.add_category(c) From 6a0fc5afd26c10aff4239d25ae1199fb32e6419b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 6 Jun 2017 21:35:57 -0400 Subject: [PATCH 091/161] Create a SQL index on History.url. This will hopefully speed up historyContains but does not seem to speed up the completion query, unfortunately. --- qutebrowser/browser/history.py | 1 + qutebrowser/misc/sql.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index d8b1f81d5..5abe8d9d6 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -78,6 +78,7 @@ class WebHistory(sql.SqlTable): def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) + self.create_index('HistoryIndex', 'url', where='not redirect') self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 0b81fc0f9..b7e180a24 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -122,6 +122,18 @@ class SqlTable(QObject): # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) + def create_index(self, name, field, where): + """Create an index over this table. + + Args: + name: Name of the index, should be unique. + field: Name of the field to index. + where: WHERE clause for a partial index. + """ + q = Query("CREATE INDEX IF NOT EXISTS {} ON {} ({}) WHERE {}" + .format(name, self._name, field, where)) + q.run() + def __iter__(self): """Iterate rows in the table.""" q = Query("SELECT * FROM {}".format(self._name)) From 5b827cf86ad2d3e1f307fd4920f9768eeaf70745 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 6 Jun 2017 21:40:17 -0400 Subject: [PATCH 092/161] Fix typo in sql exception handling --- qutebrowser/misc/sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index b7e180a24..416229ae8 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -81,8 +81,8 @@ class Query(QSqlQuery): self.addBindValue(val) log.sql.debug('self bindings: {}'.format(self.boundValues())) if not self.exec_(): - raise SqlException('Failed to exec self "{}": "{}"'.format( - self.lastself(), self.lastError().text())) + raise SqlException('Failed to exec query "{}": "{}"'.format( + self.lastQuery(), self.lastError().text())) def value(self): """Return the result of a single-value query (e.g. an EXISTS).""" From 478a719f77f99171b790020a773a8a5eee1efb5f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 6 Jun 2017 21:48:58 -0400 Subject: [PATCH 093/161] Use a prepared query for historyContains. This is called often, hopefully a prepared query will speed it up. This also modifies Query.run to return self for easier chaining, so you can use `query.run.value()` instead of `query.run` ; query.value()`. --- qutebrowser/browser/history.py | 3 ++- qutebrowser/misc/sql.py | 12 +++++------- tests/unit/misc/test_sql.py | 23 ++++++++++++++--------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 5abe8d9d6..008773e62 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -79,6 +79,7 @@ class WebHistory(sql.SqlTable): super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) self.create_index('HistoryIndex', 'url', where='not redirect') + self._contains_query = self.contains_query('url') self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' @@ -97,7 +98,7 @@ class WebHistory(sql.SqlTable): return utils.get_repr(self, length=len(self)) def __contains__(self, url): - return self.contains('url', url) + return self._contains_query.run([url]).value() def _add_entry(self, entry): """Add an entry to the in-memory database.""" diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 416229ae8..b63979c0c 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -83,6 +83,7 @@ class Query(QSqlQuery): if not self.exec_(): raise SqlException('Failed to exec query "{}": "{}"'.format( self.lastQuery(), self.lastError().text())) + return self def value(self): """Return the result of a single-value query (e.g. an EXISTS).""" @@ -140,17 +141,14 @@ class SqlTable(QObject): q.run() return iter(q) - def contains(self, field, value): - """Return whether the table contains the matching item. + def contains_query(self, field): + """Return a prepared query that checks for the existence of an item. Args: field: Field to match. - value: Value to check for the given field. """ - q = Query("Select EXISTS(SELECT * FROM {} where {} = ?)" - .format(self._name, field)) - q.run([value]) - return q.value() + return Query("Select EXISTS(SELECT * FROM {} where {} = ?)" + .format(self._name, field)) def __len__(self): """Return the count of rows in the table.""" diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index ca736b0ad..03cb0f27c 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -94,15 +94,20 @@ def test_contains(): table.insert(['one', 1, False]) table.insert(['nine', 9, False]) table.insert(['thirteen', 13, True]) - assert table.contains('name', 'one') - assert table.contains('name', 'thirteen') - assert table.contains('val', 9) - assert table.contains('lucky', False) - assert table.contains('lucky', True) - assert not table.contains('name', 'oone') - assert not table.contains('name', 1) - assert not table.contains('name', '*') - assert not table.contains('val', 10) + + name_query = table.contains_query('name') + val_query = table.contains_query('val') + lucky_query = table.contains_query('lucky') + + assert name_query.run(['one']).value() + assert name_query.run(['thirteen']).value() + assert val_query.run([9]).value() + assert lucky_query.run([False]).value() + assert lucky_query.run([True]).value() + assert not name_query.run(['oone']).value() + assert not name_query.run([1]).value() + assert not name_query.run(['*']).value() + assert not val_query.run([10]).value() def test_delete_all(qtbot): From ea0b3eee053a41c489bcf72f43a72e5ca87b6278 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 07:33:36 -0400 Subject: [PATCH 094/161] Use full, not partial index for history. historyContains includes redirect urls, so we actually don't want a partial index here. --- qutebrowser/browser/history.py | 2 +- qutebrowser/misc/sql.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 008773e62..c64674fdf 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -78,7 +78,7 @@ class WebHistory(sql.SqlTable): def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) - self.create_index('HistoryIndex', 'url', where='not redirect') + self.create_index('HistoryIndex', 'url') self._contains_query = self.contains_query('url') self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index b63979c0c..0435d2ea3 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -123,16 +123,15 @@ class SqlTable(QObject): # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) - def create_index(self, name, field, where): + def create_index(self, name, field): """Create an index over this table. Args: name: Name of the index, should be unique. field: Name of the field to index. - where: WHERE clause for a partial index. """ - q = Query("CREATE INDEX IF NOT EXISTS {} ON {} ({}) WHERE {}" - .format(name, self._name, field, where)) + q = Query("CREATE INDEX IF NOT EXISTS {} ON {} ({})" + .format(name, self._name, field)) q.run() def __iter__(self): From 309b6ba32c5918a728d6c49511fa7553902e0d0f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 08:10:44 -0400 Subject: [PATCH 095/161] Move _import_history to history.py. Also adjusts the history import test to operate at a higher level and ensure the old text file is removed (or isn't, in the case of an error). --- qutebrowser/app.py | 23 +------------------ qutebrowser/browser/history.py | 28 ++++++++++++++++++++--- tests/helpers/stubs.py | 4 ++++ tests/unit/browser/webkit/test_history.py | 18 ++++++++++----- 4 files changed, 42 insertions(+), 31 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 0f95c1c5d..ecfd456be 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -161,7 +161,7 @@ def init(args, crash_handler): QDesktopServices.setUrlHandler('https', open_desktopservices_url) QDesktopServices.setUrlHandler('qute', open_desktopservices_url) - _import_history() + objreg.get('history').import_txt() log.init.debug("Init done!") crash_handler.raise_crashdlg() @@ -479,27 +479,6 @@ def _init_modules(args, crash_handler): browsertab.init() -def _import_history(): - """Import a history text file into sqlite if it exists. - - In older versions of qutebrowser, history was stored in a text format. - This converts that file into the new sqlite format and removes it. - """ - path = os.path.join(standarddir.data(), 'history') - if not os.path.isfile(path): - return - - def action(): - with debug.log_time(log.init, 'Converting old history file to sqlite'): - objreg.get('web-history').read(path) - message.info('History import complete. Removing {}'.format(path)) - os.remove(path) - - # delay to give message time to appear before locking down for import - message.info('Converting {} to sqlite...'.format(path)) - QTimer.singleShot(100, action) - - class Quitter: """Utility class to quit/restart the QApplication. diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index c64674fdf..cb10c2bb2 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -22,10 +22,11 @@ import os import time -from PyQt5.QtCore import pyqtSlot, QUrl +from PyQt5.QtCore import pyqtSlot, QUrl, QTimer from qutebrowser.commands import cmdutils -from qutebrowser.utils import utils, objreg, log, qtutils, usertypes, message +from qutebrowser.utils import (utils, objreg, log, qtutils, usertypes, message, + debug, standarddir) from qutebrowser.misc import objects, sql @@ -217,7 +218,28 @@ class WebHistory(sql.SqlTable): return (url, title, float(atime), bool(redirect)) - def read(self, path): + def import_txt(self): + """Import a history text file into sqlite if it exists. + + In older versions of qutebrowser, history was stored in a text format. + This converts that file into the new sqlite format and removes it. + """ + path = os.path.join(standarddir.data(), 'history') + if not os.path.isfile(path): + return + + def action(): + with debug.log_time(log.init, 'Import old history file to sqlite'): + self._read(path) + message.info('History import complete. Removing {}' + .format(path)) + os.remove(path) + + # delay to give message time to appear before locking down for import + message.info('Converting {} to sqlite...'.format(path)) + QTimer.singleShot(100, action) + + def _read(self, path): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: rows = [] diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index b852426d2..717ce4c18 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -385,6 +385,10 @@ class InstaTimer(QObject): def setInterval(self, interval): pass + @staticmethod + def singleShot(_interval, fun): + fun() + class FakeConfigType: diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 16b86ca4a..5386dddd9 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -230,8 +230,9 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): assert default_interface is None -def test_read(hist, tmpdir): - histfile = tmpdir / 'history' +def test_read(hist, data_tmpdir, monkeypatch, stubs): + monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) + histfile = data_tmpdir / 'history' # empty line is deliberate, to test skipping empty lines histfile.write('''12345 http://example.com/ title 12346 http://qutebrowser.org/ @@ -239,7 +240,7 @@ def test_read(hist, tmpdir): 68891-r http://example.com/path/other ''') - hist.read(str(histfile)) + hist.import_txt() assert list(hist) == [ ('http://example.com/', 'title', 12345, False), @@ -248,6 +249,8 @@ def test_read(hist, tmpdir): ('http://example.com/path/other', '', 68891, True) ] + assert not histfile.exists() + @pytest.mark.parametrize('line', [ 'xyz http://example.com/bad-timestamp', @@ -255,12 +258,15 @@ def test_read(hist, tmpdir): 'http://example.com/no-timestamp', '68891-r-r http://example.com/double-flag', ]) -def test_read_invalid(hist, tmpdir, line): - histfile = tmpdir / 'history' +def test_read_invalid(hist, data_tmpdir, line, monkeypatch, stubs): + monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) + histfile = data_tmpdir / 'history' histfile.write(line) with pytest.raises(Exception): - hist.read(str(histfile)) + hist.import_txt() + + assert histfile.exists() def test_debug_dump_history(hist, tmpdir): From f8325cbbc1c8cade9c47ca922228fb150032af69 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 08:15:10 -0400 Subject: [PATCH 096/161] Remove print statement --- qutebrowser/completion/models/miscmodels.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 233ee8dcc..62818f767 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -137,7 +137,6 @@ def bind(_key): _key: the key being bound. """ # TODO: offer a 'Current binding' completion based on the key. - print(_key) model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) model.add_category(listcategory.ListCategory("Commands", cmdlist)) From 389e1b017895e67e93e697ffd36b0114dbe122ba Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 08:40:13 -0400 Subject: [PATCH 097/161] Fix bad objreg reference in app.py. --- qutebrowser/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index ecfd456be..8f7984f2d 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -161,7 +161,7 @@ def init(args, crash_handler): QDesktopServices.setUrlHandler('https', open_desktopservices_url) QDesktopServices.setUrlHandler('qute', open_desktopservices_url) - objreg.get('history').import_txt() + objreg.get('web-history').import_txt() log.init.debug("Init done!") crash_handler.raise_crashdlg() From 7f27603772295a22f91b5f566d1b044f5a1a0621 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 09:10:05 -0400 Subject: [PATCH 098/161] Fix columns_to_filter for sql category. --- qutebrowser/completion/models/sqlcategory.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index b8cf07ffa..c3b5fd6c2 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -61,8 +61,11 @@ class SqlCategory(QSqlQueryModel): self._query = sql.Query(querystr) self._param_count = len(filter_fields) - rec = self._query.record() - # will this work? + self.columns_to_filter = None + + # map filter_fields to indices + col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name)) + rec = col_query.run().record() self.columns_to_filter = [rec.indexOf(n) for n in filter_fields] def set_pattern(self, pattern): @@ -72,8 +75,6 @@ class SqlCategory(QSqlQueryModel): pattern: string pattern to filter by. columns_to_filter: indices of columns to apply pattern to. """ - # TODO: eliminate columns_to_filter - #assert len(columns_to_filter) == self._param_count # escape to treat a user input % or _ as a literal, not a wildcard pattern = pattern.replace('%', '\\%') pattern = pattern.replace('_', '\\_') From c64b7d00e6a793f9d34af1ac294e53a626d7fe98 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 7 Jun 2017 15:57:56 +0200 Subject: [PATCH 099/161] Add separate table for history visits --- qutebrowser/browser/history.py | 29 ++++++++++++++++---- qutebrowser/completion/models/sqlcategory.py | 2 +- qutebrowser/completion/models/urlmodel.py | 4 +-- qutebrowser/misc/sql.py | 29 ++++++++++++++++---- 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index cb10c2bb2..718ad132d 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -72,15 +72,28 @@ class Entry: return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) +class HistoryVisits(sql.SqlTable): + + """Secondary table with visited URLs and timestamps.""" + + def __init__(self, parent=None): + super().__init__("Visits", ['url', 'atime'], + fkeys={'url': 'History(url)'}) + + class WebHistory(sql.SqlTable): """The global history of visited pages.""" def __init__(self, parent=None): - super().__init__("History", ['url', 'title', 'atime', 'redirect'], + super().__init__("History", + ['url', 'title', 'last_atime', 'redirect'], + constraints={'url': 'PRIMARY KEY'}, parent=parent) + self.visits = HistoryVisits(parent=self) self.create_index('HistoryIndex', 'url') self._contains_query = self.contains_query('url') + # FIXME self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' @@ -104,7 +117,8 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, int(entry.atime), - entry.redirect]) + entry.redirect], replace=True) + self.visits.insert([entry.url_str(), int(entry.atime)]) def get_recent(self): """Get the most recent history entries.""" @@ -216,7 +230,8 @@ class WebHistory(sql.SqlTable): redirect = 'r' in flags - return (url, title, float(atime), bool(redirect)) + return ((url, float(atime)), + (url, title, float(atime), bool(redirect))) def import_txt(self): """Import a history text file into sqlite if it exists. @@ -243,17 +258,20 @@ class WebHistory(sql.SqlTable): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: rows = [] + visit_rows = [] for (i, line) in enumerate(f): line = line.strip() if not line: continue try: - row = self._parse_entry(line.strip()) + visit_row, row = self._parse_entry(line.strip()) rows.append(row) + visit_rows.append(visit_row) except ValueError: raise Exception('Failed to parse line #{} of {}: "{}"' .format(i, path, line)) - self.insert_batch(rows) + self.insert_batch(rows, replace=True) + self.visits.insert_batch(visit_rows) @cmdutils.register(instance='web-history', debug=True) def debug_dump_history(self, dest): @@ -262,6 +280,7 @@ class WebHistory(sql.SqlTable): Args: dest: Where to write the file to. """ + # FIXME dest = os.path.expanduser(dest) lines = ('{}{} {} {}' diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index c3b5fd6c2..d363d09cb 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -30,7 +30,7 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, - select='*', where=None, group_by=None, parent=None): + select='*', where=None, group_by=None, suffix=None, parent=None): """Create a new completion category backed by a sql table. Args: diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index b106da366..1be364aba 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -75,9 +75,9 @@ def url(): columns_to_filter=[0, 1])) timefmt = config.get('completion', 'timestamp-format') - select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) + select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'History', sort_order='desc', sort_by='atime', + 'History', sort_order='desc', sort_by='last_atime', filter_fields=['url', 'title'], select='url, title, {}'.format(select_time), where='not redirect') model.add_category(hist_cat) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 0435d2ea3..dec051959 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -106,7 +106,8 @@ class SqlTable(QObject): changed = pyqtSignal() - def __init__(self, name, fields, parent=None): + def __init__(self, name, fields, constraints=None, fkeys=None, + parent=None): """Create a new table in the sql database. Raises SqlException if the table already exists. @@ -114,11 +115,23 @@ class SqlTable(QObject): Args: name: Name of the table. fields: A list of field names. + constraints: A dict mapping field names to constraint strings. + fkeys: A dict mapping field names to foreign keys. """ super().__init__(parent) self._name = name + + constraints = constraints or {} + fkeys = fkeys or {} + + column_defs = ['{} {}'.format(field, constraints.get(field, '')) + for field in fields] + for field, fkey in sorted(fkeys.items()): + column_defs.append('FOREIGN KEY({}) REFERENCES {}'.format( + field, fkey)) + q = Query("CREATE TABLE IF NOT EXISTS {} ({})" - .format(name, ','.join(fields))) + .format(name, ','.join(column_defs))) q.run() # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) @@ -171,25 +184,29 @@ class SqlTable(QObject): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, values): + def insert(self, values, replace=False): """Append a row to the table. Args: values: A list of values to insert. + replace: If set, replace existing values. """ paramstr = ','.join(['?'] * len(values)) - q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q = Query("INSERT {} INTO {} values({})".format( + 'OR REPLACE' if replace else '', self._name, paramstr)) q.run(values) self.changed.emit() - def insert_batch(self, rows): + def insert_batch(self, rows, replace=False): """Performantly append multiple rows to the table. Args: rows: A list of lists, where each sub-list is a row. + replace: If set, replace existing values. """ paramstr = ','.join(['?'] * len(rows[0])) - q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q = Query("INSERT {} INTO {} values({})".format( + 'OR REPLACE' if replace else '', self._name, paramstr)) transposed = [list(row) for row in zip(*rows)] for val in transposed: From 57d96a45120e581668e7ac8dfeabd59baee8de8c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 7 Jun 2017 16:02:10 +0200 Subject: [PATCH 100/161] Add a CompletionHistory instead of HistoryVisits table --- qutebrowser/browser/history.py | 38 ++++++++++---------- qutebrowser/completion/models/sqlcategory.py | 2 +- qutebrowser/completion/models/urlmodel.py | 4 +-- qutebrowser/misc/sql.py | 11 ++---- tests/unit/completion/test_models.py | 21 +++++------ 5 files changed, 33 insertions(+), 43 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 718ad132d..5e950f111 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -72,13 +72,13 @@ class Entry: return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) -class HistoryVisits(sql.SqlTable): +class CompletionHistory(sql.SqlTable): - """Secondary table with visited URLs and timestamps.""" + """History which only has the newest entry for each URL.""" def __init__(self, parent=None): - super().__init__("Visits", ['url', 'atime'], - fkeys={'url': 'History(url)'}) + super().__init__("CompletionHistory", ['url', 'title', 'last_atime'], + constraints={'url': 'PRIMARY KEY'}, parent=parent) class WebHistory(sql.SqlTable): @@ -86,14 +86,11 @@ class WebHistory(sql.SqlTable): """The global history of visited pages.""" def __init__(self, parent=None): - super().__init__("History", - ['url', 'title', 'last_atime', 'redirect'], - constraints={'url': 'PRIMARY KEY'}, + super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) - self.visits = HistoryVisits(parent=self) + self.completion = CompletionHistory(parent=self) self.create_index('HistoryIndex', 'url') self._contains_query = self.contains_query('url') - # FIXME self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' @@ -117,8 +114,10 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, int(entry.atime), - entry.redirect], replace=True) - self.visits.insert([entry.url_str(), int(entry.atime)]) + entry.redirect]) + if not entry.redirect: + self.completion.insert([entry.url_str(), entry.title, + int(entry.atime)], replace=True) def get_recent(self): """Get the most recent history entries.""" @@ -229,9 +228,10 @@ class WebHistory(sql.SqlTable): raise ValueError("Invalid flags {!r}".format(flags)) redirect = 'r' in flags + row = (url, title, float(atime), redirect) + completion_row = None if redirect else (url, title, float(atime)) - return ((url, float(atime)), - (url, title, float(atime), bool(redirect))) + return (row, completion_row) def import_txt(self): """Import a history text file into sqlite if it exists. @@ -258,20 +258,21 @@ class WebHistory(sql.SqlTable): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: rows = [] - visit_rows = [] + completion_rows = [] for (i, line) in enumerate(f): line = line.strip() if not line: continue try: - visit_row, row = self._parse_entry(line.strip()) + row, completion_row = self._parse_entry(line.strip()) rows.append(row) - visit_rows.append(visit_row) + if completion_row is not None: + completion_rows.append(completion_row) except ValueError: raise Exception('Failed to parse line #{} of {}: "{}"' .format(i, path, line)) - self.insert_batch(rows, replace=True) - self.visits.insert_batch(visit_rows) + self.insert_batch(rows) + self.completion.insert_batch(completion_rows, replace=True) @cmdutils.register(instance='web-history', debug=True) def debug_dump_history(self, dest): @@ -280,7 +281,6 @@ class WebHistory(sql.SqlTable): Args: dest: Where to write the file to. """ - # FIXME dest = os.path.expanduser(dest) lines = ('{}{} {} {}' diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index d363d09cb..c3b5fd6c2 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -30,7 +30,7 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, - select='*', where=None, group_by=None, suffix=None, parent=None): + select='*', where=None, group_by=None, parent=None): """Create a new completion category backed by a sql table. Args: diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 1be364aba..2c92815c9 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -77,8 +77,8 @@ def url(): timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'History', sort_order='desc', sort_by='last_atime', + 'CompletionHistory', sort_order='desc', sort_by='last_atime', filter_fields=['url', 'title'], - select='url, title, {}'.format(select_time), where='not redirect') + select='url, title, {}'.format(select_time)) model.add_category(hist_cat) return model diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index dec051959..f174248b3 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -106,8 +106,7 @@ class SqlTable(QObject): changed = pyqtSignal() - def __init__(self, name, fields, constraints=None, fkeys=None, - parent=None): + def __init__(self, name, fields, constraints=None, parent=None): """Create a new table in the sql database. Raises SqlException if the table already exists. @@ -116,22 +115,16 @@ class SqlTable(QObject): name: Name of the table. fields: A list of field names. constraints: A dict mapping field names to constraint strings. - fkeys: A dict mapping field names to foreign keys. """ super().__init__(parent) self._name = name constraints = constraints or {} - fkeys = fkeys or {} - column_defs = ['{} {}'.format(field, constraints.get(field, '')) for field in fields] - for field, fkey in sorted(fkeys.items()): - column_defs.append('FOREIGN KEY({}) REFERENCES {}'.format( - field, fkey)) - q = Query("CREATE TABLE IF NOT EXISTS {} ({})" .format(name, ','.join(column_defs))) + q.run() # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 46730cc21..c85ecbf38 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -155,24 +155,22 @@ def bookmarks(bookmark_manager_stub): @pytest.fixture def web_history_stub(stubs, init_sql): - return sql.SqlTable("History", ['url', 'title', 'atime', 'redirect']) + return sql.SqlTable("CompletionHistory", ['url', 'title', 'last_atime']) @pytest.fixture def web_history(web_history_stub, init_sql): """Pre-populate the web-history database.""" - web_history_stub.insert(['http://some-redirect.example.com', 'redirect', - datetime(2016, 9, 5).timestamp(), True]) web_history_stub.insert(['http://qutebrowser.org', 'qutebrowser', - datetime(2015, 9, 5).timestamp(), False]) + datetime(2015, 9, 5).timestamp()]) web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2016, 2, 8).timestamp(), False]) + datetime(2016, 2, 8).timestamp()]) web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2016, 3, 8).timestamp(), False]) + datetime(2016, 3, 8).timestamp()]) web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2014, 3, 8).timestamp(), False]) + datetime(2014, 3, 8).timestamp()]) web_history_stub.insert(['https://github.com', 'https://github.com', - datetime(2016, 5, 1).timestamp(), False]) + datetime(2016, 5, 1).timestamp()]) return web_history_stub @@ -336,7 +334,7 @@ def test_url_completion_pattern(config_stub, web_history_stub, url, title, pattern, rowcount): """Test that url completion filters by url and title.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} - web_history_stub.insert([url, title, 0, False]) + web_history_stub.insert([url, title, 0]) model = urlmodel.url() model.set_pattern(pattern) # 2, 0 is History @@ -582,10 +580,9 @@ def test_url_completion_benchmark(benchmark, config_stub, 'web-history-max-items': 1000} entries = [web_history_stub.Entry( - atime=i, + last_atime=i, url='http://example.com/{}'.format(i), - title='title{}'.format(i), - redirect=False) + title='title{}'.format(i)) for i in range(100000)] web_history_stub.insert_batch(entries) From 6ce52f39ae28be0aec4243c236b2a03b6a0d5b9d Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 7 Jun 2017 16:37:52 +0200 Subject: [PATCH 101/161] Add debug timings for SQL --- qutebrowser/browser/webkit/webkithistory.py | 5 ++++- qutebrowser/completion/models/sqlcategory.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/webkit/webkithistory.py b/qutebrowser/browser/webkit/webkithistory.py index 453a11883..64b7bf295 100644 --- a/qutebrowser/browser/webkit/webkithistory.py +++ b/qutebrowser/browser/webkit/webkithistory.py @@ -22,6 +22,8 @@ from PyQt5.QtWebKit import QWebHistoryInterface +from qutebrowser.utils import debug + class WebHistoryInterface(QWebHistoryInterface): @@ -48,7 +50,8 @@ class WebHistoryInterface(QWebHistoryInterface): Return: True if the url is in the history, False otherwise. """ - return url_string in self._history + with debug.log_time('sql', 'historyContains'): + return url_string in self._history def init(history): diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index c3b5fd6c2..e2801b072 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -24,6 +24,7 @@ import re from PyQt5.QtSql import QSqlQueryModel from qutebrowser.misc import sql +from qutebrowser.utils import debug class SqlCategory(QSqlQueryModel): @@ -81,5 +82,6 @@ class SqlCategory(QSqlQueryModel): # treat spaces as wildcards to match any of the typed words pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) - self._query.run([pattern] * self._param_count) + with debug.log_time('sql', 'Running completion query'): + self._query.run([pattern] * self._param_count) self.setQuery(self._query) From feed9c8936aaf086bf54eb2ce41a1a260d6cc973 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 20:49:11 -0400 Subject: [PATCH 102/161] Better exception handling in history. - Show an error message when import fails, not a generic crash dialog - Raise CommandError when debug-dump-history fails - Check that the path exists for debug-dump-history --- qutebrowser/browser/history.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 5e950f111..62c773866 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -24,7 +24,7 @@ import time from PyQt5.QtCore import pyqtSlot, QUrl, QTimer -from qutebrowser.commands import cmdutils +from qutebrowser.commands import cmdutils, cmdexc from qutebrowser.utils import (utils, objreg, log, qtutils, usertypes, message, debug, standarddir) from qutebrowser.misc import objects, sql @@ -245,10 +245,14 @@ class WebHistory(sql.SqlTable): def action(): with debug.log_time(log.init, 'Import old history file to sqlite'): - self._read(path) - message.info('History import complete. Removing {}' - .format(path)) - os.remove(path) + try: + self._read(path) + except ValueError as ex: + message.error('Failed to import history: {}'.format(ex)) + else: + message.info('History import complete. Removing {}' + .format(path)) + os.remove(path) # delay to give message time to appear before locking down for import message.info('Converting {} to sqlite...'.format(path)) @@ -268,9 +272,9 @@ class WebHistory(sql.SqlTable): rows.append(row) if completion_row is not None: completion_rows.append(completion_row) - except ValueError: - raise Exception('Failed to parse line #{} of {}: "{}"' - .format(i, path, line)) + except ValueError as ex: + raise ValueError('Failed to parse line #{} of {}: "{}"' + .format(i, path, ex)) self.insert_batch(rows) self.completion.insert_batch(completion_rows, replace=True) @@ -283,6 +287,10 @@ class WebHistory(sql.SqlTable): """ dest = os.path.expanduser(dest) + dirname = os.path.dirname(dest) + if not os.path.exists(dirname): + raise cmdexc.CommandError('Path does not exist', dirname) + lines = ('{}{} {} {}' .format(int(x.atime), '-r' * x.redirect, x.url, x.title) for x in self.select(sort_by='atime', sort_order='asc')) @@ -291,9 +299,8 @@ class WebHistory(sql.SqlTable): try: f.write('\n'.join(lines)) except OSError as e: - message.error('Could not write history: {}'.format(e)) - else: - message.info("Dumped history to {}.".format(dest)) + raise cmdexc.CommandError('Could not write history: {}', e) + message.info("Dumped history to {}.".format(dest)) def init(parent=None): From 6fc61d12fc538c4e129b1e3f31f2cfc8dee6bfdf Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 22:45:14 -0400 Subject: [PATCH 103/161] Assorted small fixes for sql code review. --- qutebrowser/browser/qutescheme.py | 2 +- qutebrowser/completion/completionwidget.py | 19 +++++++------ .../completion/models/completionmodel.py | 12 ++++---- qutebrowser/completion/models/listcategory.py | 3 +- qutebrowser/completion/models/sqlcategory.py | 6 ++-- qutebrowser/completion/models/urlmodel.py | 3 +- qutebrowser/misc/sql.py | 28 +++++++++++++------ tests/end2end/features/misc.feature | 2 -- tests/end2end/features/test_history_bdd.py | 19 ++++++------- tests/helpers/utils.py | 13 +++++++++ tests/unit/browser/test_qutescheme.py | 22 --------------- tests/unit/browser/webkit/test_history.py | 20 ++++--------- .../unit/completion/test_completionwidget.py | 4 +-- tests/unit/completion/test_listcategory.py | 16 ++--------- tests/unit/completion/test_sqlcategory.py | 24 ++++------------ tests/unit/utils/test_version.py | 2 +- 16 files changed, 83 insertions(+), 112 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 4ebc626d3..0c495d28b 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -213,7 +213,7 @@ def qute_history(url): offset = QUrlQuery(url).queryItemValue("offset") offset = int(offset) if offset else None except ValueError as e: - raise QuteSchemeError("Query parameter start_time is invalid", e) + raise QuteSchemeError("Query parameter offset is invalid", e) # Use start_time in query or current time. try: start_time = QUrlQuery(url).queryItemValue("start_time") diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 711301b4c..f4d4d80b7 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -111,7 +111,6 @@ class CompletionView(QTreeView): # objreg.get('config').changed.connect(self.init_command_completion) objreg.get('config').changed.connect(self._on_config_changed) - self._column_widths = (30, 70, 0) self._active = False self._delegate = completiondelegate.CompletionItemDelegate(self) @@ -150,7 +149,8 @@ class CompletionView(QTreeView): def _resize_columns(self): """Resize the completion columns based on column_widths.""" width = self.size().width() - pixel_widths = [(width * perc // 100) for perc in self._column_widths] + column_widths = self.model.column_widths + pixel_widths = [(width * perc // 100) for perc in column_widths] if self.verticalScrollBar().isVisible(): delta = self.style().pixelMetric(QStyle.PM_ScrollBarExtent) + 5 @@ -280,14 +280,8 @@ class CompletionView(QTreeView): model.setParent(self) self.setModel(model) - self._column_widths = model.column_widths self._active = True - - if (config.get('completion', 'show') == 'always' and - model.count() > 0): - self.show() - else: - self.hide() + self._maybe_show() for i in range(model.rowCount()): self.expand(model.index(i, 0)) @@ -297,7 +291,14 @@ class CompletionView(QTreeView): self.model().set_pattern(pattern) self._resize_columns() self._maybe_update_geometry() + self._maybe_show() + + def _maybe_show(self): + if (config.get('completion', 'show') == 'always' and + model.count() > 0): self.show() + else: + self.hide() def _maybe_update_geometry(self): """Emit the update_geometry signal if the config says so.""" diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index cbc20470c..2dec0ed46 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -78,12 +78,14 @@ class CompletionModel(QAbstractItemModel): if not index.isValid() or role != Qt.DisplayRole: return None if not index.parent().isValid(): + # category header if index.column() == 0: return self._categories[index.row()].name - else: - cat = self._categories[index.parent().row()] - idx = cat.index(index.row(), index.column()) - return cat.data(idx) + return None + # item + cat = self._categories[index.parent().row()] + idx = cat.index(index.row(), index.column()) + return cat.data(idx) def flags(self, index): """Return the item flags for index. @@ -132,7 +134,7 @@ class CompletionModel(QAbstractItemModel): # categories have no parent return QModelIndex() row = self._categories.index(parent_cat) - return self.createIndex(row, 0, None) + return self.createIndex(row, index.column(), None) def rowCount(self, parent=QModelIndex()): """Override QAbstractItemModel::rowCount.""" diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index 75cc2642c..b22d795be 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -71,8 +71,7 @@ class ListCategory(QSortFilterProxyModel): parent: The parent item QModelIndex. Return: - True if self.pattern is contained in item, or if it's a root item - (category). False in all other cases + True if self.pattern is contained in item. """ if not self.pattern: return True diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index e2801b072..2060f383f 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -28,6 +28,7 @@ from qutebrowser.utils import debug class SqlCategory(QSqlQueryModel): + """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, @@ -57,12 +58,11 @@ class SqlCategory(QSqlQueryModel): if group_by: querystr += ' group by {}'.format(group_by) if sort_by: - assert sort_order in ['asc', 'desc'] + assert sort_order in ['asc', 'desc'], sort_order querystr += ' order by {} {}'.format(sort_by, sort_order) - self._query = sql.Query(querystr) + self._query = sql.Query(querystr, forward_only=False) self._param_count = len(filter_fields) - self.columns_to_filter = None # map filter_fields to indices col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name)) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 2c92815c9..7e42285a6 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -74,7 +74,8 @@ def url(): model.add_category(listcategory.ListCategory('Bookmarks', bookmarks, columns_to_filter=[0, 1])) - timefmt = config.get('completion', 'timestamp-format') + # replace 's to avoid breaking the query + timefmt = config.get('completion', 'timestamp-format').replace("'", "`") select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( 'CompletionHistory', sort_order='desc', sort_by='last_atime', diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index f174248b3..98eebba43 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -59,13 +59,23 @@ class Query(QSqlQuery): """A prepared SQL Query.""" - def __init__(self, querystr): + def __init__(self, querystr, forward_only=True): + """Prepare a new sql query. + + Args: + querystr: String to prepare query from. + forward_only: Optimization for queries that will only step forward. + Must be false for completion queries. + """ super().__init__(QSqlDatabase.database()) log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) - self.prepare(querystr) + if not self.prepare(querystr): + raise SqlException('Failed to prepare query "{}"'.format(querystr)) + self.setForwardOnly(forward_only) def __iter__(self): - assert self.isActive(), "Cannot iterate inactive query" + if not self.isActive(): + raise SqlException("Cannot iterate inactive query") rec = self.record() fields = [rec.fieldName(i) for i in range(rec.count())] rowtype = collections.namedtuple('ResultRow', fields) @@ -87,8 +97,8 @@ class Query(QSqlQuery): def value(self): """Return the result of a single-value query (e.g. an EXISTS).""" - ok = self.next() - assert ok, "No result for single-result query" + if not self.next(): + raise SqlException("No result for single-result query") return self.record().value(0) @@ -152,7 +162,7 @@ class SqlTable(QObject): Args: field: Field to match. """ - return Query("Select EXISTS(SELECT * FROM {} where {} = ?)" + return Query("SELECT EXISTS(SELECT * FROM {} WHERE {} = ?)" .format(self._name, field)) def __len__(self): @@ -214,17 +224,19 @@ class SqlTable(QObject): self.changed.emit() def delete_all(self): - """Remove all row from the table.""" + """Remove all rows from the table.""" Query("DELETE FROM {}".format(self._name)).run() self.changed.emit() def select(self, sort_by, sort_order, limit=-1): - """Remove all row from the table. + """Prepare, run, and return a select statement on this table. Args: sort_by: name of column to sort by. sort_order: 'asc' or 'desc'. limit: max number of rows in result, defaults to -1 (unlimited). + + Return: A prepared and executed select query. """ q = Query('SELECT * FROM {} ORDER BY {} {} LIMIT ?' .format(self._name, sort_by, sort_order)) diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index f6c5ce84a..1a1c0c95b 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -702,8 +702,6 @@ Feature: Various utility commands. And I wait for "Renderer process was killed" in the log And I open data/numbers/3.txt Then no crash should happen - And the following tabs should be open: - - data/numbers/3.txt (active) ## Other diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 197bc3d20..5249e891c 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -28,17 +28,16 @@ bdd.scenarios('history.feature') @bdd.then(bdd.parsers.parse("the history should contain:\n{expected}")) -def check_history(quteproc, expected, httpbin): - with tempfile.TemporaryDirectory() as tmpdir: - path = os.path.join(tmpdir, 'history') - quteproc.send_cmd(':debug-dump-history "{}"'.format(path)) - quteproc.wait_for(category='message', loglevel=logging.INFO, - message='Dumped history to {}.'.format(path)) +def check_history(quteproc, httpbin, tmpdir, expected): + path = tmpdir / 'history' + quteproc.send_cmd(':debug-dump-history "{}"'.format(path)) + quteproc.wait_for(category='message', loglevel=logging.INFO, + message='Dumped history to {}.'.format(path)) - with open(path, 'r', encoding='utf-8') as f: - # ignore access times, they will differ in each run - actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() - for line in f.read().splitlines()) + with open(path, 'r', encoding='utf-8') as f: + # ignore access times, they will differ in each run + actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() + for line in f.read()) expected = expected.replace('(port)', str(httpbin.port)) assert actual == expected diff --git a/tests/helpers/utils.py b/tests/helpers/utils.py index 7dbd7dd25..051b7bd50 100644 --- a/tests/helpers/utils.py +++ b/tests/helpers/utils.py @@ -170,3 +170,16 @@ def abs_datapath(): """Get the absolute path to the end2end data directory.""" file_abs = os.path.abspath(os.path.dirname(__file__)) return os.path.join(file_abs, '..', 'end2end', 'data') + + +def validate_model(cat, expected): + """Check that a category contains the expected items in the given order. + + Args: + cat: The category to inspect. + expected: A list of tuples containing the expected items. + """ + assert cat.rowCount() == len(expected) + for row, items in enumerate(expected): + for col, item in enumerate(items): + assert cat.data(cat.index(row, col)) == item diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 0f9f43373..87d0662b5 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -122,7 +122,6 @@ class TestHistoryHandler: url = QUrl("qute://history/data?start_time=" + str(start_time)) _mimetype, data = qutescheme.qute_history(url) items = json.loads(data) - items = [item for item in items if 'time' in item] # skip 'next' item assert len(items) == expected_item_count @@ -132,27 +131,6 @@ class TestHistoryHandler: assert item['time'] <= start_time * 1000 assert item['time'] > end_time * 1000 - @pytest.mark.skip("TODO: do we need next?") - @pytest.mark.parametrize("start_time_offset, next_time", [ - (0, 24*60*60), - (24*60*60, 48*60*60), - (48*60*60, -1), - (72*60*60, -1) - ]) - def test_qutehistory_next(self, start_time_offset, next_time, now): - """Ensure qute://history/data returns correct items.""" - start_time = now - start_time_offset - url = QUrl("qute://history/data?start_time=" + str(start_time)) - _mimetype, data = qutescheme.qute_history(url) - items = json.loads(data) - items = [item for item in items if 'next' in item] # 'next' items - assert len(items) == 1 - - if next_time == -1: - assert items[0]["next"] == -1 - else: - assert items[0]["next"] == now - next_time - def test_qute_history_benchmark(self, fake_web_history, benchmark, now): entries = [] for t in range(100000): # one history per second diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 5386dddd9..841da4ea7 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -107,14 +107,6 @@ def test_entries_before(hist): assert times == [12348, 12347, 12346] -def test_save(tmpdir, hist): - hist.add_url(QUrl('http://example.com/'), atime=12345) - hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) - - hist2 = history.WebHistory() - assert list(hist2) == [('http://example.com/', '', 12345, False), - ('http://www.qutebrowser.org/', '', 67890, False)] - def test_clear(qtbot, tmpdir, hist, mocker): hist.add_url(QUrl('http://example.com/')) @@ -132,12 +124,11 @@ def test_clear_force(qtbot, tmpdir, hist): assert not len(hist) -@pytest.mark.parametrize('item', [ +@pytest.mark.parametrize('url, atime, title, redirect', [ ('http://www.example.com', 12346, 'the title', False), ('http://www.example.com', 12346, 'the title', True) ]) -def test_add_item(qtbot, hist, item): - (url, atime, title, redirect) = item +def test_add_item(qtbot, hist, url, atime, title, redirect): hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) assert list(hist) == [(url, title, atime, redirect)] @@ -188,13 +179,14 @@ def test_history_interface(qtbot, webview, hist_interface): def cleanup_init(): # prevent test_init from leaking state yield - try: - hist = objreg.get('web-history') + hist = objreg.get('web-history', None) + if hist is not None: hist.setParent(None) objreg.delete('web-history') + try: from PyQt5.QtWebKit import QWebHistoryInterface QWebHistoryInterface.setDefaultInterface(None) - except: + except Exception: pass diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 66f6e7f89..4cc59deea 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -229,7 +229,7 @@ def test_completion_item_del_no_selection(completionview): model = completionmodel.CompletionModel(delete_cur_item=func) model.add_category(listcategory.ListCategory('', [('foo',)])) completionview.set_model(model) - with pytest.raises(cmdexc.CommandError): + with pytest.raises(cmdexc.CommandError, match='No item selected!'): completionview.completion_item_del() assert not func.called @@ -240,5 +240,5 @@ def test_completion_item_del_no_func(completionview): model.add_category(listcategory.ListCategory('', [('foo',)])) completionview.set_model(model) completionview.completion_item_focus('next') - with pytest.raises(cmdexc.CommandError): + with pytest.raises(cmdexc.CommandError, match='Cannot delete this item.'): completionview.completion_item_del() diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index a0c300473..d8d35ea3c 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -21,22 +21,10 @@ import pytest +from helpers import utils from qutebrowser.completion.models import listcategory -def _validate(cat, expected): - """Check that a category contains the expected items in the given order. - - Args: - cat: The category to inspect. - expected: A list of tuples containing the expected items. - """ - assert cat.rowCount() == len(expected) - for row, items in enumerate(expected): - for col, item in enumerate(items): - assert cat.data(cat.index(row, col)) == item - - @pytest.mark.parametrize('pattern, filter_cols, before, after', [ ('foo', [0], [('foo', '', ''), ('bar', '', '')], @@ -68,4 +56,4 @@ def test_set_pattern(pattern, filter_cols, before, after): cat = listcategory.ListCategory('Foo', before, columns_to_filter=filter_cols) cat.set_pattern(pattern) - _validate(cat, after) + utils.validate_model(cat, after) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 1b4190d21..2a5b73445 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -21,6 +21,7 @@ import pytest +from helpers import utils from qutebrowser.misc import sql from qutebrowser.completion.models import sqlcategory @@ -28,19 +29,6 @@ from qutebrowser.completion.models import sqlcategory pytestmark = pytest.mark.usefixtures('init_sql') -def _validate(cat, expected): - """Check that a category contains the expected items in the given order. - - Args: - cat: The category to inspect. - expected: A list of tuples containing the expected items. - """ - assert cat.rowCount() == len(expected) - for row, items in enumerate(expected): - for col, item in enumerate(items): - assert cat.data(cat.index(row, col)) == item - - @pytest.mark.parametrize('sort_by, sort_order, data, expected', [ (None, 'asc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], @@ -77,7 +65,7 @@ def test_sorting(sort_by, sort_order, data, expected): cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) cat.set_pattern('') - _validate(cat, expected) + utils.validate_model(cat, expected) @pytest.mark.parametrize('pattern, filter_cols, before, after', [ @@ -133,7 +121,7 @@ def test_set_pattern(pattern, filter_cols, before, after): filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) cat.set_pattern(pattern) - _validate(cat, after) + utils.validate_model(cat, after) def test_select(): @@ -141,7 +129,7 @@ def test_select(): table.insert(['foo', 'bar', 'baz']) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') cat.set_pattern('') - _validate(cat, [('bar', 'baz', 'foo')]) + utils.validate_model(cat, [('bar', 'baz', 'foo')]) def test_where(): @@ -150,7 +138,7 @@ def test_where(): table.insert(['baz', 'biz', True]) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') cat.set_pattern('') - _validate(cat, [('foo', 'bar', False)]) + utils.validate_model(cat, [('foo', 'bar', False)]) def test_group(): @@ -162,7 +150,7 @@ def test_group(): cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='a, max(b)', group_by='a') cat.set_pattern('') - _validate(cat, [('bar', 3), ('foo', 2)]) + utils.validate_model(cat, [('bar', 3), ('foo', 2)]) def test_entry(): diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 11bf326fb..13dfb09d2 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -798,7 +798,6 @@ def test_chromium_version_unpatched(qapp): assert version._chromium_version() not in ['', 'unknown', 'unavailable'] -# pylint: disable=too-many-locals @pytest.mark.parametrize(['git_commit', 'frozen', 'style', 'with_webkit', 'known_distribution'], [ (True, False, True, True, True), # normal @@ -812,6 +811,7 @@ def test_chromium_version_unpatched(qapp): def test_version_output(git_commit, frozen, style, with_webkit, known_distribution, stubs, monkeypatch, init_sql): """Test version.version().""" + # pylint: disable=too-many-locals class FakeWebEngineProfile: def httpUserAgent(self): return 'Toaster/4.0.4 Chrome/CHROMIUMVERSION Teapot/4.1.8' From 9b0395db087f320e150626df7e182c66f9a47c86 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 8 Jun 2017 13:03:44 +0200 Subject: [PATCH 104/161] Add an lru cache for WebHistoryInterface.historyContains When loading heise.de, for some crazy reason QtWebKit calls historyContains about 16'000 times. With this cache (which we simply clear when *any* page has been loaded, as then the links which have been visited can change), that's down to 250 or so... --- qutebrowser/browser/webkit/webkithistory.py | 3 +++ qutebrowser/misc/utilcmds.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/qutebrowser/browser/webkit/webkithistory.py b/qutebrowser/browser/webkit/webkithistory.py index 64b7bf295..0edbb3fa3 100644 --- a/qutebrowser/browser/webkit/webkithistory.py +++ b/qutebrowser/browser/webkit/webkithistory.py @@ -19,6 +19,7 @@ """QtWebKit specific part of history.""" +import functools from PyQt5.QtWebKit import QWebHistoryInterface @@ -36,11 +37,13 @@ class WebHistoryInterface(QWebHistoryInterface): def __init__(self, webhistory, parent=None): super().__init__(parent) self._history = webhistory + self._history.changed.connect(self.historyContains.cache_clear) def addHistoryEntry(self, url_string): """Required for a QWebHistoryInterface impl, obsoleted by add_url.""" pass + @functools.lru_cache(maxsize=32768) def historyContains(self, url_string): """Called by WebKit to determine if a URL is contained in the history. diff --git a/qutebrowser/misc/utilcmds.py b/qutebrowser/misc/utilcmds.py index 41b44de1f..d1771c212 100644 --- a/qutebrowser/misc/utilcmds.py +++ b/qutebrowser/misc/utilcmds.py @@ -170,8 +170,15 @@ def debug_cache_stats(): """Print LRU cache stats.""" config_info = objreg.get('config').get.cache_info() style_info = style.get_stylesheet.cache_info() + try: + from PyQt5.QtWebKit import QWebHistoryInterface + interface = QWebHistoryInterface.defaultInterface() + history_info = interface.historyContains.cache_info() + except ImportError: + history_info = None log.misc.debug('config: {}'.format(config_info)) log.misc.debug('style: {}'.format(style_info)) + log.misc.debug('history: {}'.format(history_info)) @cmdutils.register(debug=True) From 3a4ef09f58312b3bc614150950ad662ebf92f2dd Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 07:34:35 -0400 Subject: [PATCH 105/161] More sql code review fixes --- qutebrowser/completion/completionwidget.py | 4 ++-- tests/end2end/features/test_history_bdd.py | 4 ++-- tests/unit/browser/webkit/test_history.py | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index f4d4d80b7..15f317c5a 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -149,7 +149,7 @@ class CompletionView(QTreeView): def _resize_columns(self): """Resize the completion columns based on column_widths.""" width = self.size().width() - column_widths = self.model.column_widths + column_widths = self.model().column_widths pixel_widths = [(width * perc // 100) for perc in column_widths] if self.verticalScrollBar().isVisible(): @@ -295,7 +295,7 @@ class CompletionView(QTreeView): def _maybe_show(self): if (config.get('completion', 'show') == 'always' and - model.count() > 0): + self.model().count() > 0): self.show() else: self.hide() diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 5249e891c..75f20efcb 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -44,5 +44,5 @@ def check_history(quteproc, httpbin, tmpdir, expected): @bdd.then("the history should be empty") -def check_history_empty(quteproc, httpbin): - check_history(quteproc, '', httpbin) +def check_history_empty(quteproc, httpbin, tmpdir): + check_history(quteproc, httpbin, tmpdir, '') diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 841da4ea7..49b499534 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -107,7 +107,6 @@ def test_entries_before(hist): assert times == [12348, 12347, 12346] - def test_clear(qtbot, tmpdir, hist, mocker): hist.add_url(QUrl('http://example.com/')) hist.add_url(QUrl('http://www.qutebrowser.org/')) @@ -186,7 +185,7 @@ def cleanup_init(): try: from PyQt5.QtWebKit import QWebHistoryInterface QWebHistoryInterface.setDefaultInterface(None) - except Exception: + except ImportError: pass From 18cd8ba0b6dc3bcfa68f77614fc2b59396968f6d Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 8 Jun 2017 13:58:36 +0200 Subject: [PATCH 106/161] Add indices for HistoryAtimeIndex and CompletionHistoryAtimeIndex before ------ sqlite> SELECT * FROM History where not redirect and not url like "qute://%" and atime > ? and atime <= ? ORDER BY atime desc; Run Time: real 0.072 user 0.063334 sys 0.010000 sqlite> explain query plan SELECT * FROM History where not redirect and not url like "qute://%" and atime > ? and atime <= ? ORDER BY atime desc; 0|0|0|SCAN TABLE History 0|0|0|USE TEMP B-TREE FOR ORDER BY sqlite> explain query plan select url, title, strftime('%Y-%m-%d', last_atime, 'unixepoch') from CompletionHistory where (url like "%qute%" or title like "%qute%") order by last_atime desc; 0|0|0|SCAN TABLE CompletionHistory 0|0|0|USE TEMP B-TREE FOR ORDER BY after ----- sqlite> SELECT * FROM History where not redirect and not url like "qute://%" and atime > ? and atime <= ? ORDER BY atime desc; Run Time: real 0.000 user 0.000000 sys 0.000000 sqlite> explain query plan SELECT * FROM History where not redirect and not url like "qute://%" and atime > ? and atime <= ? ORDER BY atime desc; 0|0|0|SEARCH TABLE History USING INDEX AtimeIndex (atime>? AND atime explain query plan select url, title, strftime('%Y-%m-%d', last_atime, 'unixepoch') from CompletionHistory where (url like "%qute%" or title like "%qute%") order by last_atime desc; 0|0|0|SCAN TABLE CompletionHistory USING INDEX CompletionAtimeIndex --- qutebrowser/browser/history.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 62c773866..05f2fefd5 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -79,6 +79,7 @@ class CompletionHistory(sql.SqlTable): def __init__(self, parent=None): super().__init__("CompletionHistory", ['url', 'title', 'last_atime'], constraints={'url': 'PRIMARY KEY'}, parent=parent) + self.create_index('CompletionHistoryAtimeIndex', 'last_atime') class WebHistory(sql.SqlTable): @@ -90,6 +91,7 @@ class WebHistory(sql.SqlTable): parent=parent) self.completion = CompletionHistory(parent=self) self.create_index('HistoryIndex', 'url') + self.create_index('HistoryAtimeIndex', 'atime') self._contains_query = self.contains_query('url') self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' From 862f8d318846a6dfdd49dc7a754ffb8c56be6289 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 08:43:53 -0400 Subject: [PATCH 107/161] Always return col 0 for index parent. This was changed during code review but was causing Qt errors while TAB-completing in the selection view: 08:42:34 WARNING qt Unknown module:none:0 Can't select indexes from different model or with different parents --- qutebrowser/completion/models/completionmodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 2dec0ed46..2d74d94e0 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -134,7 +134,7 @@ class CompletionModel(QAbstractItemModel): # categories have no parent return QModelIndex() row = self._categories.index(parent_cat) - return self.createIndex(row, index.column(), None) + return self.createIndex(row, 0, None) def rowCount(self, parent=QModelIndex()): """Override QAbstractItemModel::rowCount.""" From 22b7b21d5a5d0d8db4243a3d8fd9a831f85c7cce Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 21:43:00 -0400 Subject: [PATCH 108/161] Use named placeholders for sql queries. --- qutebrowser/browser/history.py | 23 +++--- qutebrowser/completion/models/sqlcategory.py | 4 +- qutebrowser/misc/sql.py | 37 ++++++---- tests/unit/completion/test_sqlcategory.py | 18 ++--- tests/unit/misc/test_sql.py | 76 ++++++++++++-------- 5 files changed, 92 insertions(+), 66 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 05f2fefd5..0b9b7db81 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -96,30 +96,31 @@ class WebHistory(sql.SqlTable): self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' - 'and atime > ? ' - 'and atime <= ? ' + 'and atime > :earliest ' + 'and atime <= :latest ' 'ORDER BY atime desc') self._before_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' - 'and atime <= ? ' + 'and atime <= :latest ' 'ORDER BY atime desc ' - 'limit ? offset ?') + 'limit :limit offset :offset') def __repr__(self): return utils.get_repr(self, length=len(self)) def __contains__(self, url): - return self._contains_query.run([url]).value() + return self._contains_query.run(val=url).value() def _add_entry(self, entry): """Add an entry to the in-memory database.""" - self.insert([entry.url_str(), entry.title, int(entry.atime), - entry.redirect]) + self.insert(url=entry.url_str(), title=entry.title, + atime=int(entry.atime), redirect=entry.redirect) if not entry.redirect: - self.completion.insert([entry.url_str(), entry.title, - int(entry.atime)], replace=True) + self.completion.insert_or_replace(url=entry.url_str(), + title=entry.title, + last_atime=int(entry.atime)) def get_recent(self): """Get the most recent history entries.""" @@ -132,7 +133,7 @@ class WebHistory(sql.SqlTable): earliest: Omit timestamps earlier than this. latest: Omit timestamps later than this. """ - self._between_query.run([earliest, latest]) + self._between_query.run(earliest=earliest, latest=latest) return iter(self._between_query) def entries_before(self, latest, limit, offset): @@ -143,7 +144,7 @@ class WebHistory(sql.SqlTable): limit: Max number of entries to include. offset: Number of entries to skip. """ - self._before_query.run([latest, limit, offset]) + self._before_query.run(latest=latest, limit=limit, offset=offset) return iter(self._before_query) @cmdutils.register(name='history-clear', instance='web-history') diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 2060f383f..d49b1db51 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -49,7 +49,7 @@ class SqlCategory(QSqlQueryModel): querystr = 'select {} from {} where ('.format(select, name) # the incoming pattern will have literal % and _ escaped with '\' # we need to tell sql to treat '\' as an escape character - querystr += ' or '.join("{} like ? escape '\\'".format(f) + querystr += ' or '.join("{} like :pattern escape '\\'".format(f) for f in filter_fields) querystr += ')' @@ -83,5 +83,5 @@ class SqlCategory(QSqlQueryModel): pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) with debug.log_time('sql', 'Running completion query'): - self._query.run([pattern] * self._param_count) + self._query.run(pattern=pattern) self.setQuery(self._query) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 98eebba43..5c3e94c1e 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -84,11 +84,11 @@ class Query(QSqlQuery): rec = self.record() yield rowtype(*[rec.value(i) for i in range(rec.count())]) - def run(self, values=None): + def run(self, **values): """Execute the prepared query.""" log.sql.debug('Running SQL query: "{}"'.format(self.lastQuery())) - for val in values or []: - self.addBindValue(val) + for key, val in values.items(): + self.bindValue(':{}'.format(key), val) log.sql.debug('self bindings: {}'.format(self.boundValues())) if not self.exec_(): raise SqlException('Failed to exec query "{}": "{}"'.format( @@ -162,7 +162,7 @@ class SqlTable(QObject): Args: field: Field to match. """ - return Query("SELECT EXISTS(SELECT * FROM {} WHERE {} = ?)" + return Query("SELECT EXISTS(SELECT * FROM {} WHERE {} = :val)" .format(self._name, field)) def __len__(self): @@ -181,23 +181,34 @@ class SqlTable(QObject): Return: The number of rows deleted. """ - q = Query("DELETE FROM {} where {} = ?".format(self._name, field)) - q.run([value]) + q = Query("DELETE FROM {} where {} = :val".format(self._name, field)) + q.run(val=value) if not q.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, values, replace=False): + def insert(self, **values): """Append a row to the table. Args: values: A list of values to insert. replace: If set, replace existing values. """ - paramstr = ','.join(['?'] * len(values)) - q = Query("INSERT {} INTO {} values({})".format( - 'OR REPLACE' if replace else '', self._name, paramstr)) - q.run(values) + paramstr = ','.join(':{}'.format(key) for key in values.keys()) + q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q.run(**values) + self.changed.emit() + + def insert_or_replace(self, **values): + """Append a row to the table. + + Args: + values: A list of values to insert. + replace: If set, replace existing values. + """ + paramstr = ','.join(':{}'.format(key) for key in values.keys()) + q = Query("REPLACE INTO {} values({})".format(self._name, paramstr)) + q.run(**values) self.changed.emit() def insert_batch(self, rows, replace=False): @@ -238,7 +249,7 @@ class SqlTable(QObject): Return: A prepared and executed select query. """ - q = Query('SELECT * FROM {} ORDER BY {} {} LIMIT ?' + q = Query('SELECT * FROM {} ORDER BY {} {} LIMIT :limit' .format(self._name, sort_by, sort_order)) - q.run([limit]) + q.run(limit=limit) return q diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 2a5b73445..cef17c964 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -61,7 +61,7 @@ pytestmark = pytest.mark.usefixtures('init_sql') def test_sorting(sort_by, sort_order, data, expected): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: - table.insert(row) + table.insert(a=row[0], b=row[1], c=row[2]) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) cat.set_pattern('') @@ -117,7 +117,7 @@ def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: - table.insert(row) + table.insert(a=row[0], b=row[1], c=row[2]) filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) cat.set_pattern(pattern) @@ -126,7 +126,7 @@ def test_set_pattern(pattern, filter_cols, before, after): def test_select(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert(['foo', 'bar', 'baz']) + table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') cat.set_pattern('') utils.validate_model(cat, [('bar', 'baz', 'foo')]) @@ -134,8 +134,8 @@ def test_select(): def test_where(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert(['foo', 'bar', False]) - table.insert(['baz', 'biz', True]) + table.insert({'a': 'foo', 'b': 'bar', 'c': False}) + table.insert({'a': 'baz', 'b': 'biz', 'c': True}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') cat.set_pattern('') utils.validate_model(cat, [('foo', 'bar', False)]) @@ -143,10 +143,10 @@ def test_where(): def test_group(): table = sql.SqlTable('Foo', ['a', 'b']) - table.insert(['foo', 1]) - table.insert(['bar', 3]) - table.insert(['foo', 2]) - table.insert(['bar', 0]) + table.insert({'a': 'foo', 'b': 1}) + table.insert({'a': 'bar', 'b': 3}) + table.insert({'a': 'foo', 'b': 2}) + table.insert({'a': 'bar', 'b': 0}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='a, max(b)', group_by='a') cat.set_pattern('') diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 03cb0f27c..3216400f0 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -35,39 +35,53 @@ def test_init(): def test_insert(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) with qtbot.waitSignal(table.changed): - table.insert(['one', 1, False]) + table.insert(name='one', val=1, lucky=False) with qtbot.waitSignal(table.changed): - table.insert(['wan', 1, False]) + table.insert(name='wan', val=1, lucky=False) + + +def test_insert_or_replace(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], + constraints={'name': 'PRIMARY KEY'}) + with qtbot.waitSignal(table.changed): + table.insert_or_replace(name='one', val=1, lucky=False) + with qtbot.waitSignal(table.changed): + table.insert_or_replace(name='one', val=11, lucky=True) + assert list(table) == [('one', 11, True)] def test_iter(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) + table.insert(name='one', val=1, lucky=False) + table.insert(name='nine', val=9, lucky=False) + table.insert(name='thirteen', val=13, lucky=True) assert list(table) == [('one', 1, False), ('nine', 9, False), ('thirteen', 13, True)] @pytest.mark.parametrize('rows, sort_by, sort_order, limit, result', [ - ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', 5, [(1, 6), (2, 5), (3, 4)]), - ([[2, 5], [1, 6], [3, 4]], 'a', 'desc', 3, [(3, 4), (2, 5), (1, 6)]), - ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]), - ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', -1, [(1, 6), (2, 5), (3, 4)]), + ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'a', 'asc', 5, + [(1, 6), (2, 5), (3, 4)]), + ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'a', 'desc', 3, + [(3, 4), (2, 5), (1, 6)]), + ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'b', 'desc', 2, + [(1, 6), (2, 5)]), + ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'a', 'asc', -1, + [(1, 6), (2, 5), (3, 4)]), ]) def test_select(rows, sort_by, sort_order, limit, result): table = sql.SqlTable('Foo', ['a', 'b']) for row in rows: - table.insert(row) + table.insert(**row) assert list(table.select(sort_by, sort_order, limit)) == result def test_delete(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) + table.insert(name='one', val=1, lucky=False) + table.insert(name='nine', val=9, lucky=False) + table.insert(name='thirteen', val=13, lucky=True) with pytest.raises(KeyError): table.delete('nope', 'name') with qtbot.waitSignal(table.changed): @@ -81,40 +95,40 @@ def test_delete(qtbot): def test_len(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) assert len(table) == 0 - table.insert(['one', 1, False]) + table.insert(name='one', val=1, lucky=False) assert len(table) == 1 - table.insert(['nine', 9, False]) + table.insert(name='nine', val=9, lucky=False) assert len(table) == 2 - table.insert(['thirteen', 13, True]) + table.insert(name='thirteen', val=13, lucky=True) assert len(table) == 3 def test_contains(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) + table.insert(name='one', val=1, lucky=False) + table.insert(name='nine', val=9, lucky=False) + table.insert(name='thirteen', val=13, lucky=True) name_query = table.contains_query('name') val_query = table.contains_query('val') lucky_query = table.contains_query('lucky') - assert name_query.run(['one']).value() - assert name_query.run(['thirteen']).value() - assert val_query.run([9]).value() - assert lucky_query.run([False]).value() - assert lucky_query.run([True]).value() - assert not name_query.run(['oone']).value() - assert not name_query.run([1]).value() - assert not name_query.run(['*']).value() - assert not val_query.run([10]).value() + assert name_query.run(val='one').value() + assert name_query.run(val='thirteen').value() + assert val_query.run(val=9).value() + assert lucky_query.run(val=False).value() + assert lucky_query.run(val=True).value() + assert not name_query.run(val='oone').value() + assert not name_query.run(val=1).value() + assert not name_query.run(val='*').value() + assert not val_query.run(val=10).value() def test_delete_all(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) + table.insert(name='one', val=1, lucky=False) + table.insert(name='nine', val=9, lucky=False) + table.insert(name='thirteen', val=13, lucky=True) with qtbot.waitSignal(table.changed): table.delete_all() assert list(table) == [] From a6a9ad72f91e9401c37535045a320da305dc48e9 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 22:18:34 -0400 Subject: [PATCH 109/161] Fix test_history_interface. This was still using a history dict instead of SQL history. --- tests/unit/browser/webkit/test_history.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 49b499534..f99df5587 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -152,7 +152,7 @@ def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog): @pytest.fixture -def hist_interface(): +def hist_interface(hist): # pylint: disable=invalid-name QtWebKit = pytest.importorskip('PyQt5.QtWebKit') from qutebrowser.browser.webkit import webkithistory @@ -160,8 +160,7 @@ def hist_interface(): # pylint: enable=invalid-name entry = history.Entry(atime=0, url=QUrl('http://www.example.com/'), title='example') - history_dict = {'http://www.example.com/': entry} - interface = webkithistory.WebHistoryInterface(history_dict) + interface = webkithistory.WebHistoryInterface(hist) QWebHistoryInterface.setDefaultInterface(interface) yield QWebHistoryInterface.setDefaultInterface(None) From fa39b82b3c7d6ffa5f04453b3f1bffe7811f15d4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 22:18:52 -0400 Subject: [PATCH 110/161] Backup old history file after import. Instead of removing it, move it to history.bak. --- qutebrowser/browser/history.py | 10 ++++++---- tests/unit/browser/webkit/test_history.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 0b9b7db81..4739e56c8 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -240,7 +240,8 @@ class WebHistory(sql.SqlTable): """Import a history text file into sqlite if it exists. In older versions of qutebrowser, history was stored in a text format. - This converts that file into the new sqlite format and removes it. + This converts that file into the new sqlite format and moves it to a + backup location. """ path = os.path.join(standarddir.data(), 'history') if not os.path.isfile(path): @@ -253,9 +254,10 @@ class WebHistory(sql.SqlTable): except ValueError as ex: message.error('Failed to import history: {}'.format(ex)) else: - message.info('History import complete. Removing {}' - .format(path)) - os.remove(path) + bakpath = path + '.bak' + message.info('History import complete. Moving {} to {}' + .format(path, bakpath)) + os.rename(path, bakpath) # delay to give message time to appear before locking down for import message.info('Converting {} to sqlite...'.format(path)) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index f99df5587..d87f30b38 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -240,6 +240,7 @@ def test_read(hist, data_tmpdir, monkeypatch, stubs): ] assert not histfile.exists() + assert (data_tmpdir / 'history.bak').exists() @pytest.mark.parametrize('line', [ From 61a1709141013a60c159d99b1b4919d611919be1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 22:27:46 -0400 Subject: [PATCH 111/161] Fix completion selection bug. Fix the issue where pressing `oo` would show a url completion dialog where attempting to select items would do nothing but show a Qt warning. The fix is to ensure we set _last_completion_func to None whenever we clear completion (there was a case I missed). It also ensures we always delete the old model and adds a safety to prevent deleting an in-use model is set_model is called with the current model. --- qutebrowser/completion/completer.py | 1 + qutebrowser/completion/completionwidget.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 491fe2e04..ae72add20 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -211,6 +211,7 @@ class Completer(QObject): # FIXME complete searches # https://github.com/qutebrowser/qutebrowser/issues/32 completion.set_model(None) + self._last_completion_func = None return before_cursor, pattern, after_cursor = self._partition() diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 15f317c5a..7ea4edf13 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -269,17 +269,18 @@ class CompletionView(QTreeView): Args: model: The model to use. """ + if self.model() is not None and model is not self.model(): + self.model().deleteLater() + self.selectionModel().deleteLater() + + self.setModel(model) + if model is None: self._active = False self.hide() return - if self.model() is not None: - self.model().deleteLater() - self.selectionModel().deleteLater() - model.setParent(self) - self.setModel(model) self._active = True self._maybe_show() From 679e001a48d763b8fed7788ad4f4ccef6900d906 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 9 Jun 2017 08:06:12 -0400 Subject: [PATCH 112/161] Separate sqlcategory title from table name. Also fix a number of sql/completion tests that were failing. --- qutebrowser/completion/models/sqlcategory.py | 10 +++++---- qutebrowser/completion/models/urlmodel.py | 3 ++- tests/unit/completion/test_models.py | 23 +++++++++----------- tests/unit/completion/test_sqlcategory.py | 14 ++++++------ 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index d49b1db51..7e0ecf2c5 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -31,12 +31,14 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" - def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, - select='*', where=None, group_by=None, parent=None): + def __init__(self, name, *, title=None, filter_fields, sort_by=None, + sort_order=None, select='*', where=None, group_by=None, + parent=None): """Create a new completion category backed by a sql table. Args: - name: Name of category, and the table in the database. + name: Name of the table in the database. + title: Title of category, defaults to table name. filter_fields: Names of fields to apply filter pattern to. select: A custom result column expression for the select statement. where: An optional clause to filter out some rows. @@ -44,7 +46,7 @@ class SqlCategory(QSqlQueryModel): sort_order: Either 'asc' or 'desc', if sort_by is non-None """ super().__init__(parent=parent) - self.name = name + self.name = title or name querystr = 'select {} from {} where ('.format(select, name) # the incoming pattern will have literal % and _ escaped with '\' diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 7e42285a6..8cacca873 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -78,7 +78,8 @@ def url(): timefmt = config.get('completion', 'timestamp-format').replace("'", "`") select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'CompletionHistory', sort_order='desc', sort_by='last_atime', + 'CompletionHistory', title='History', + sort_order='desc', sort_by='last_atime', filter_fields=['url', 'title'], select='url, title, {}'.format(select_time)) model.add_category(hist_cat) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index c85ecbf38..89da30013 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -161,16 +161,15 @@ def web_history_stub(stubs, init_sql): @pytest.fixture def web_history(web_history_stub, init_sql): """Pre-populate the web-history database.""" - web_history_stub.insert(['http://qutebrowser.org', 'qutebrowser', - datetime(2015, 9, 5).timestamp()]) - web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2016, 2, 8).timestamp()]) - web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2016, 3, 8).timestamp()]) - web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2014, 3, 8).timestamp()]) - web_history_stub.insert(['https://github.com', 'https://github.com', - datetime(2016, 5, 1).timestamp()]) + web_history_stub.insert(url='http://qutebrowser.org', + title='qutebrowser', + last_atime=datetime(2015, 9, 5).timestamp()) + web_history_stub.insert(url='https://python.org', + title='Welcome to Python.org', + last_atime=datetime(2016, 3, 8).timestamp()) + web_history_stub.insert(url='https://github.com', + title='https://github.com', + last_atime=datetime(2016, 5, 1).timestamp()) return web_history_stub @@ -304,9 +303,7 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, "History": [ ('https://github.com', 'https://github.com', '2016-05-01'), ('https://python.org', 'Welcome to Python.org', '2016-03-08'), - ('https://python.org', 'Welcome to Python.org', '2016-02-08'), ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), - ('https://python.org', 'Welcome to Python.org', '2014-03-08'), ], }) @@ -334,7 +331,7 @@ def test_url_completion_pattern(config_stub, web_history_stub, url, title, pattern, rowcount): """Test that url completion filters by url and title.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} - web_history_stub.insert([url, title, 0]) + web_history_stub.insert(url=url, title=title, last_atime=0) model = urlmodel.url() model.set_pattern(pattern) # 2, 0 is History diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index cef17c964..d81fa8cf0 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -126,7 +126,7 @@ def test_set_pattern(pattern, filter_cols, before, after): def test_select(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) + table.insert(a='foo', b='bar', c='baz') cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') cat.set_pattern('') utils.validate_model(cat, [('bar', 'baz', 'foo')]) @@ -134,8 +134,8 @@ def test_select(): def test_where(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert({'a': 'foo', 'b': 'bar', 'c': False}) - table.insert({'a': 'baz', 'b': 'biz', 'c': True}) + table.insert(a='foo', b='bar', c=False) + table.insert(a='baz', b='biz', c=True) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') cat.set_pattern('') utils.validate_model(cat, [('foo', 'bar', False)]) @@ -143,10 +143,10 @@ def test_where(): def test_group(): table = sql.SqlTable('Foo', ['a', 'b']) - table.insert({'a': 'foo', 'b': 1}) - table.insert({'a': 'bar', 'b': 3}) - table.insert({'a': 'foo', 'b': 2}) - table.insert({'a': 'bar', 'b': 0}) + table.insert(a='foo', b=1) + table.insert(a='bar', b=3) + table.insert(a='foo', b=2) + table.insert(a='bar', b=0) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='a, max(b)', group_by='a') cat.set_pattern('') From cf23f42b99f99c170c2a42511aabf53d6b51ae82 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 9 Jun 2017 12:07:39 -0400 Subject: [PATCH 113/161] Use splitlines in test_history_bdd again. Just using read() returns a single string, and iterating over that iterates over each character. --- tests/end2end/features/test_history_bdd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 75f20efcb..ce9faeca9 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -37,7 +37,7 @@ def check_history(quteproc, httpbin, tmpdir, expected): with open(path, 'r', encoding='utf-8') as f: # ignore access times, they will differ in each run actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() - for line in f.read()) + for line in f.read().splitlines()) expected = expected.replace('(port)', str(httpbin.port)) assert actual == expected From 1fe18134319e59763bfad93f8e43f8568cf87f3e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 10 Jun 2017 07:53:52 -0400 Subject: [PATCH 114/161] Fix pylint errors. --- qutebrowser/app.py | 2 +- qutebrowser/completion/models/listcategory.py | 2 +- qutebrowser/completion/models/sqlcategory.py | 1 - qutebrowser/misc/sql.py | 4 ++-- tests/end2end/features/test_history_bdd.py | 2 -- tests/unit/browser/webkit/test_history.py | 3 +-- 6 files changed, 5 insertions(+), 9 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 8f7984f2d..a8fd7ad54 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -57,7 +57,7 @@ from qutebrowser.misc import (readline, ipc, savemanager, sessions, crashsignal, earlyinit, objects, sql) from qutebrowser.misc import utilcmds # pylint: disable=unused-import from qutebrowser.utils import (log, version, message, utils, qtutils, urlutils, - objreg, usertypes, standarddir, error, debug) + objreg, usertypes, standarddir, error) # We import utilcmds to run the cmdutils.register decorators. diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index b22d795be..acdc5cafc 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -28,7 +28,7 @@ import re from PyQt5.QtCore import QSortFilterProxyModel from PyQt5.QtGui import QStandardItem, QStandardItemModel -from qutebrowser.utils import qtutils, debug, log +from qutebrowser.utils import qtutils class ListCategory(QSortFilterProxyModel): diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 7e0ecf2c5..9e88b52da 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -64,7 +64,6 @@ class SqlCategory(QSqlQueryModel): querystr += ' order by {} {}'.format(sort_by, sort_order) self._query = sql.Query(querystr, forward_only=False) - self._param_count = len(filter_fields) # map filter_fields to indices col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name)) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 5c3e94c1e..27b4d4268 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -194,7 +194,7 @@ class SqlTable(QObject): values: A list of values to insert. replace: If set, replace existing values. """ - paramstr = ','.join(':{}'.format(key) for key in values.keys()) + paramstr = ','.join(':{}'.format(key) for key in values) q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) q.run(**values) self.changed.emit() @@ -206,7 +206,7 @@ class SqlTable(QObject): values: A list of values to insert. replace: If set, replace existing values. """ - paramstr = ','.join(':{}'.format(key) for key in values.keys()) + paramstr = ','.join(':{}'.format(key) for key in values) q = Query("REPLACE INTO {} values({})".format(self._name, paramstr)) q.run(**values) self.changed.emit() diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index ce9faeca9..c1000d6ce 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -18,9 +18,7 @@ # along with qutebrowser. If not, see . import logging -import os.path import re -import tempfile import pytest_bdd as bdd diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index d87f30b38..04081e7c7 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -158,8 +158,7 @@ def hist_interface(hist): from qutebrowser.browser.webkit import webkithistory QWebHistoryInterface = QtWebKit.QWebHistoryInterface # pylint: enable=invalid-name - entry = history.Entry(atime=0, url=QUrl('http://www.example.com/'), - title='example') + hist.add_url(url=QUrl('http://www.example.com/'), title='example') interface = webkithistory.WebHistoryInterface(hist) QWebHistoryInterface.setDefaultInterface(interface) yield From f4f52ee204513f0c4d0c9a596977a2f6c00494e6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 10 Jun 2017 13:51:11 -0400 Subject: [PATCH 115/161] Remove history.Entry. No longer needed with sql backend. Query results build their own namedtuple from the returned columns, and inserting new entries is just done with named parameters. --- qutebrowser/browser/history.py | 62 +++------------------------ tests/unit/browser/test_qutescheme.py | 7 +-- 2 files changed, 11 insertions(+), 58 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 4739e56c8..6965cf3ea 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -30,48 +30,6 @@ from qutebrowser.utils import (utils, objreg, log, qtutils, usertypes, message, from qutebrowser.misc import objects, sql -class Entry: - - """A single entry in the web history. - - Attributes: - atime: The time the page was accessed. - url: The URL which was accessed as QUrl. - redirect: If True, don't show this entry in completion - """ - - def __init__(self, atime, url, title, redirect=False): - self.atime = float(atime) - self.url = url - self.title = title - self.redirect = redirect - qtutils.ensure_valid(url) - - def __repr__(self): - return utils.get_repr(self, constructor=True, atime=self.atime, - url=self.url_str(), title=self.title, - redirect=self.redirect) - - def __str__(self): - atime = str(int(self.atime)) - if self.redirect: - atime += '-r' # redirect flag - elems = [atime, self.url_str()] - if self.title: - elems.append(self.title) - return ' '.join(elems) - - def __eq__(self, other): - return (self.atime == other.atime and - self.title == other.title and - self.url == other.url and - self.redirect == other.redirect) - - def url_str(self): - """Get the URL as a lossless string.""" - return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) - - class CompletionHistory(sql.SqlTable): """History which only has the newest entry for each URL.""" @@ -113,15 +71,6 @@ class WebHistory(sql.SqlTable): def __contains__(self, url): return self._contains_query.run(val=url).value() - def _add_entry(self, entry): - """Add an entry to the in-memory database.""" - self.insert(url=entry.url_str(), title=entry.title, - atime=int(entry.atime), redirect=entry.redirect) - if not entry.redirect: - self.completion.insert_or_replace(url=entry.url_str(), - title=entry.title, - last_atime=int(entry.atime)) - def get_recent(self): """Get the most recent history entries.""" return self.select(sort_by='atime', sort_order='desc', limit=100) @@ -199,10 +148,13 @@ class WebHistory(sql.SqlTable): log.misc.warning("Ignoring invalid URL being added to history") return - if atime is None: - atime = time.time() - entry = Entry(atime, url, title, redirect=redirect) - self._add_entry(entry) + atime = int(atime) if (atime is not None) else int(time.time()) + url_str = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) + self.insert(url=url_str, title=title, atime=atime, redirect=redirect) + if not redirect: + self.completion.insert_or_replace(url=url_str, + title=title, + last_atime=atime) def _parse_entry(self, line): """Parse a history line like '12345 http://example.com title'.""" diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 87d0662b5..fefafc9a1 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -89,8 +89,9 @@ class TestHistoryHandler: items = [] for i in range(entry_count): entry_atime = now - i * interval - entry = history.Entry(atime=str(entry_atime), - url=QUrl("www.x.com/" + str(i)), title="Page " + str(i)) + entry = {"atime": str(entry_atime), + "url": QUrl("www.x.com/" + str(i)), + "title": "Page " + str(i)} items.insert(0, entry) return items @@ -107,7 +108,7 @@ class TestHistoryHandler: def fake_history(self, fake_web_history, entries): """Create fake history.""" for item in entries: - fake_web_history._add_entry(item) + fake_web_history.add_url(**item) @pytest.mark.parametrize("start_time_offset, expected_item_count", [ (0, 4), From e436f4816493de4314a0f48fa256096bdb0aa1f4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 10 Jun 2017 18:41:24 -0400 Subject: [PATCH 116/161] Small sql fixes. - Remove unused SqlTable.Entry - Fix wording of two log messages - Remove unused import --- qutebrowser/browser/history.py | 4 ++-- qutebrowser/misc/sql.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 6965cf3ea..75c184198 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -25,7 +25,7 @@ import time from PyQt5.QtCore import pyqtSlot, QUrl, QTimer from qutebrowser.commands import cmdutils, cmdexc -from qutebrowser.utils import (utils, objreg, log, qtutils, usertypes, message, +from qutebrowser.utils import (utils, objreg, log, usertypes, message, debug, standarddir) from qutebrowser.misc import objects, sql @@ -257,7 +257,7 @@ class WebHistory(sql.SqlTable): f.write('\n'.join(lines)) except OSError as e: raise cmdexc.CommandError('Could not write history: {}', e) - message.info("Dumped history to {}.".format(dest)) + message.info("Dumped history to {}".format(dest)) def init(parent=None): diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 27b4d4268..301ffb2ff 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -89,7 +89,7 @@ class Query(QSqlQuery): log.sql.debug('Running SQL query: "{}"'.format(self.lastQuery())) for key, val in values.items(): self.bindValue(':{}'.format(key), val) - log.sql.debug('self bindings: {}'.format(self.boundValues())) + log.sql.debug('query bindings: {}'.format(self.boundValues())) if not self.exec_(): raise SqlException('Failed to exec query "{}": "{}"'.format( self.lastQuery(), self.lastError().text())) @@ -107,7 +107,6 @@ class SqlTable(QObject): """Interface to a sql table. Attributes: - Entry: The class wrapping row data from this table. _name: Name of the SQL table this wraps. Signals: @@ -136,8 +135,6 @@ class SqlTable(QObject): .format(name, ','.join(column_defs))) q.run() - # pylint: disable=invalid-name - self.Entry = collections.namedtuple(name + '_Entry', fields) def create_index(self, name, field): """Create an index over this table. From 4e87773d895b4c14e52231c0b4f997ea9e7933d3 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 11 Jun 2017 13:57:38 -0400 Subject: [PATCH 117/161] Use a dict instead of named params for insert. This allows replace to be a named parameter and allows consolidating some duplicate code between various insert methods. This also fixes some tests that broke because batch insert was broken. --- qutebrowser/browser/history.py | 38 ++++++------ qutebrowser/misc/sql.py | 38 +++++------- tests/unit/browser/test_qutescheme.py | 15 +++-- tests/unit/completion/test_models.py | 43 +++++++------- tests/unit/misc/test_sql.py | 83 ++++++++++++++++++++------- 5 files changed, 126 insertions(+), 91 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 75c184198..e314f6c5e 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -150,11 +150,15 @@ class WebHistory(sql.SqlTable): atime = int(atime) if (atime is not None) else int(time.time()) url_str = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) - self.insert(url=url_str, title=title, atime=atime, redirect=redirect) + self.insert({'url': url_str, + 'title': title, + 'atime': atime, + 'redirect': redirect}) if not redirect: - self.completion.insert_or_replace(url=url_str, - title=title, - last_atime=atime) + self.completion.insert({'url': url_str, + 'title': title, + 'last_atime': atime}, + replace=True) def _parse_entry(self, line): """Parse a history line like '12345 http://example.com title'.""" @@ -183,10 +187,7 @@ class WebHistory(sql.SqlTable): raise ValueError("Invalid flags {!r}".format(flags)) redirect = 'r' in flags - row = (url, title, float(atime), redirect) - completion_row = None if redirect else (url, title, float(atime)) - - return (row, completion_row) + return (url, title, int(atime), redirect) def import_txt(self): """Import a history text file into sqlite if it exists. @@ -218,22 +219,27 @@ class WebHistory(sql.SqlTable): def _read(self, path): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: - rows = [] - completion_rows = [] + data = {'url': [], 'title': [], 'atime': [], 'redirect': []} + completion_data = {'url': [], 'title': [], 'last_atime': []} for (i, line) in enumerate(f): line = line.strip() if not line: continue try: - row, completion_row = self._parse_entry(line.strip()) - rows.append(row) - if completion_row is not None: - completion_rows.append(completion_row) + url, title, atime, redirect = self._parse_entry(line) + data['url'].append(url) + data['title'].append(title) + data['atime'].append(atime) + data['redirect'].append(redirect) + if not redirect: + completion_data['url'].append(url) + completion_data['title'].append(title) + completion_data['last_atime'].append(atime) except ValueError as ex: raise ValueError('Failed to parse line #{} of {}: "{}"' .format(i, path, ex)) - self.insert_batch(rows) - self.completion.insert_batch(completion_rows, replace=True) + self.insert_batch(data) + self.completion.insert_batch(completion_data, replace=True) @cmdutils.register(instance='web-history', debug=True) def debug_dump_history(self, dest): diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 301ffb2ff..717ae3c0f 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -184,44 +184,32 @@ class SqlTable(QObject): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, **values): + def _insert_query(self, values, replace): + params = ','.join(':{}'.format(key) for key in values) + verb = "REPLACE" if replace else "INSERT" + return Query("{} INTO {} values({})".format(verb, self._name, params)) + + def insert(self, values, replace=False): """Append a row to the table. Args: - values: A list of values to insert. + values: A dict with a value to insert for each field name. replace: If set, replace existing values. """ - paramstr = ','.join(':{}'.format(key) for key in values) - q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q = self._insert_query(values, replace) q.run(**values) self.changed.emit() - def insert_or_replace(self, **values): - """Append a row to the table. - - Args: - values: A list of values to insert. - replace: If set, replace existing values. - """ - paramstr = ','.join(':{}'.format(key) for key in values) - q = Query("REPLACE INTO {} values({})".format(self._name, paramstr)) - q.run(**values) - self.changed.emit() - - def insert_batch(self, rows, replace=False): + def insert_batch(self, values, replace=False): """Performantly append multiple rows to the table. Args: rows: A list of lists, where each sub-list is a row. - replace: If set, replace existing values. + values: A dict with a list of values to insert for each field name. """ - paramstr = ','.join(['?'] * len(rows[0])) - q = Query("INSERT {} INTO {} values({})".format( - 'OR REPLACE' if replace else '', self._name, paramstr)) - - transposed = [list(row) for row in zip(*rows)] - for val in transposed: - q.addBindValue(val) + q = self._insert_query(values, replace) + for key, val in values.items(): + q.bindValue(':{}'.format(key), val) db = QSqlDatabase.database() db.transaction() diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index fefafc9a1..154af0355 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -133,14 +133,13 @@ class TestHistoryHandler: assert item['time'] > end_time * 1000 def test_qute_history_benchmark(self, fake_web_history, benchmark, now): - entries = [] - for t in range(100000): # one history per second - entry = fake_web_history.Entry( - atime=str(now - t), - url=QUrl('www.x.com/{}'.format(t)), - title='x at {}'.format(t), - redirect=False) - entries.append(entry) + r = range(100000) + entries = { + 'atime': [int(now - t) for t in r], + 'url': ['www.x.com/{}'.format(t) for t in r], + 'title': ['x at {}'.format(t) for t in r], + 'redirect': [False for _ in r], + } fake_web_history.insert_batch(entries) url = QUrl("qute://history/data?start_time={}".format(now)) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 89da30013..e9e8e6366 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -161,15 +161,15 @@ def web_history_stub(stubs, init_sql): @pytest.fixture def web_history(web_history_stub, init_sql): """Pre-populate the web-history database.""" - web_history_stub.insert(url='http://qutebrowser.org', - title='qutebrowser', - last_atime=datetime(2015, 9, 5).timestamp()) - web_history_stub.insert(url='https://python.org', - title='Welcome to Python.org', - last_atime=datetime(2016, 3, 8).timestamp()) - web_history_stub.insert(url='https://github.com', - title='https://github.com', - last_atime=datetime(2016, 5, 1).timestamp()) + web_history_stub.insert({'url': 'http://qutebrowser.org', + 'title': 'qutebrowser', + 'last_atime': datetime(2015, 9, 5).timestamp()}) + web_history_stub.insert({'url': 'https://python.org', + 'title': 'Welcome to Python.org', + 'last_atime': datetime(2016, 3, 8).timestamp()}) + web_history_stub.insert({'url': 'https://github.com', + 'title': 'https://github.com', + 'last_atime': datetime(2016, 5, 1).timestamp()}) return web_history_stub @@ -331,7 +331,7 @@ def test_url_completion_pattern(config_stub, web_history_stub, url, title, pattern, rowcount): """Test that url completion filters by url and title.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} - web_history_stub.insert(url=url, title=title, last_atime=0) + web_history_stub.insert({'url': url, 'title': title, 'last_atime': 0}) model = urlmodel.url() model.set_pattern(pattern) # 2, 0 is History @@ -576,21 +576,22 @@ def test_url_completion_benchmark(benchmark, config_stub, config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 1000} - entries = [web_history_stub.Entry( - last_atime=i, - url='http://example.com/{}'.format(i), - title='title{}'.format(i)) - for i in range(100000)] + r = range(100000) + entries = { + 'last_atime': list(r), + 'url': ['http://example.com/{}'.format(i) for i in r], + 'title': ['title{}'.format(i) for i in r] + } web_history_stub.insert_batch(entries) - quickmark_manager_stub.marks = collections.OrderedDict( - (e.title, e.url) - for e in entries[0:1000]) + quickmark_manager_stub.marks = collections.OrderedDict([ + ('title{}'.format(i), 'example.com/{}'.format(i)) + for i in range(1000)]) - bookmark_manager_stub.marks = collections.OrderedDict( - (e.url, e.title) - for e in entries[0:1000]) + bookmark_manager_stub.marks = collections.OrderedDict([ + ('example.com/{}'.format(i), 'title{}'.format(i)) + for i in range(1000)]) def bench(): model = urlmodel.url() diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 3216400f0..afd952255 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -35,26 +35,67 @@ def test_init(): def test_insert(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) with qtbot.waitSignal(table.changed): - table.insert(name='one', val=1, lucky=False) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) with qtbot.waitSignal(table.changed): - table.insert(name='wan', val=1, lucky=False) + table.insert({'name': 'wan', 'val': 1, 'lucky': False}) -def test_insert_or_replace(qtbot): +def test_insert_replace(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], constraints={'name': 'PRIMARY KEY'}) with qtbot.waitSignal(table.changed): - table.insert_or_replace(name='one', val=1, lucky=False) + table.insert({'name': 'one', 'val': 1, 'lucky': False}, replace=True) with qtbot.waitSignal(table.changed): - table.insert_or_replace(name='one', val=11, lucky=True) + table.insert({'name': 'one', 'val': 11, 'lucky': True}, replace=True) assert list(table) == [('one', 11, True)] + with pytest.raises(sql.SqlException): + table.insert({'name': 'one', 'val': 11, 'lucky': True}, replace=False) + + +def test_insert_batch(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) + + with qtbot.waitSignal(table.changed): + table.insert_batch({'name': ['one', 'nine', 'thirteen'], + 'val': [1, 9, 13], + 'lucky': [False, False, True]}) + + assert list(table) == [('one', 1, False), + ('nine', 9, False), + ('thirteen', 13, True)] + + +def test_insert_batch_replace(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], + constraints={'name': 'PRIMARY KEY'}) + + with qtbot.waitSignal(table.changed): + table.insert_batch({'name': ['one', 'nine', 'thirteen'], + 'val': [1, 9, 13], + 'lucky': [False, False, True]}) + + with qtbot.waitSignal(table.changed): + table.insert_batch({'name': ['one', 'nine'], + 'val': [11, 19], + 'lucky': [True, True]}, + replace=True) + + assert list(table) == [('thirteen', 13, True), + ('one', 11, True), + ('nine', 19, True)] + + with pytest.raises(sql.SqlException): + table.insert_batch({'name': ['one', 'nine'], + 'val': [11, 19], + 'lucky': [True, True]}) + def test_iter(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(name='one', val=1, lucky=False) - table.insert(name='nine', val=9, lucky=False) - table.insert(name='thirteen', val=13, lucky=True) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) + table.insert({'name': 'nine', 'val': 9, 'lucky': False}) + table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) assert list(table) == [('one', 1, False), ('nine', 9, False), ('thirteen', 13, True)] @@ -73,15 +114,15 @@ def test_iter(): def test_select(rows, sort_by, sort_order, limit, result): table = sql.SqlTable('Foo', ['a', 'b']) for row in rows: - table.insert(**row) + table.insert(row) assert list(table.select(sort_by, sort_order, limit)) == result def test_delete(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(name='one', val=1, lucky=False) - table.insert(name='nine', val=9, lucky=False) - table.insert(name='thirteen', val=13, lucky=True) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) + table.insert({'name': 'nine', 'val': 9, 'lucky': False}) + table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) with pytest.raises(KeyError): table.delete('nope', 'name') with qtbot.waitSignal(table.changed): @@ -95,19 +136,19 @@ def test_delete(qtbot): def test_len(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) assert len(table) == 0 - table.insert(name='one', val=1, lucky=False) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) assert len(table) == 1 - table.insert(name='nine', val=9, lucky=False) + table.insert({'name': 'nine', 'val': 9, 'lucky': False}) assert len(table) == 2 - table.insert(name='thirteen', val=13, lucky=True) + table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) assert len(table) == 3 def test_contains(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(name='one', val=1, lucky=False) - table.insert(name='nine', val=9, lucky=False) - table.insert(name='thirteen', val=13, lucky=True) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) + table.insert({'name': 'nine', 'val': 9, 'lucky': False}) + table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) name_query = table.contains_query('name') val_query = table.contains_query('val') @@ -126,9 +167,9 @@ def test_contains(): def test_delete_all(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(name='one', val=1, lucky=False) - table.insert(name='nine', val=9, lucky=False) - table.insert(name='thirteen', val=13, lucky=True) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) + table.insert({'name': 'nine', 'val': 9, 'lucky': False}) + table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) with qtbot.waitSignal(table.changed): table.delete_all() assert list(table) == [] From c7a18a8b8d9ff723216d4535257e22e821d67aaf Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 13 Jun 2017 21:14:47 -0400 Subject: [PATCH 118/161] Fix tests for recent sql changes --- tests/end2end/features/test_history_bdd.py | 2 +- tests/unit/completion/test_sqlcategory.py | 25 ++++++++-------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index c1000d6ce..f5639f5f3 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -30,7 +30,7 @@ def check_history(quteproc, httpbin, tmpdir, expected): path = tmpdir / 'history' quteproc.send_cmd(':debug-dump-history "{}"'.format(path)) quteproc.wait_for(category='message', loglevel=logging.INFO, - message='Dumped history to {}.'.format(path)) + message='Dumped history to {}'.format(path)) with open(path, 'r', encoding='utf-8') as f: # ignore access times, they will differ in each run diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index d81fa8cf0..ad1ff7e75 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -61,7 +61,7 @@ pytestmark = pytest.mark.usefixtures('init_sql') def test_sorting(sort_by, sort_order, data, expected): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: - table.insert(a=row[0], b=row[1], c=row[2]) + table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) cat.set_pattern('') @@ -117,7 +117,7 @@ def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: - table.insert(a=row[0], b=row[1], c=row[2]) + table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) cat.set_pattern(pattern) @@ -126,7 +126,7 @@ def test_set_pattern(pattern, filter_cols, before, after): def test_select(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert(a='foo', b='bar', c='baz') + table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') cat.set_pattern('') utils.validate_model(cat, [('bar', 'baz', 'foo')]) @@ -134,8 +134,8 @@ def test_select(): def test_where(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert(a='foo', b='bar', c=False) - table.insert(a='baz', b='biz', c=True) + table.insert({'a': 'foo', 'b': 'bar', 'c': False}) + table.insert({'a': 'baz', 'b': 'biz', 'c': True}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') cat.set_pattern('') utils.validate_model(cat, [('foo', 'bar', False)]) @@ -143,18 +143,11 @@ def test_where(): def test_group(): table = sql.SqlTable('Foo', ['a', 'b']) - table.insert(a='foo', b=1) - table.insert(a='bar', b=3) - table.insert(a='foo', b=2) - table.insert(a='bar', b=0) + table.insert({'a': 'foo', 'b': 1}) + table.insert({'a': 'bar', 'b': 3}) + table.insert({'a': 'foo', 'b': 2}) + table.insert({'a': 'bar', 'b': 0}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='a, max(b)', group_by='a') cat.set_pattern('') utils.validate_model(cat, [('bar', 3), ('foo', 2)]) - - -def test_entry(): - table = sql.SqlTable('Foo', ['a', 'b', 'c']) - assert hasattr(table.Entry, 'a') - assert hasattr(table.Entry, 'b') - assert hasattr(table.Entry, 'c') From 891a6bcf1437156f95212bb3052c336e934e8147 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 14 Jun 2017 07:15:42 -0400 Subject: [PATCH 119/161] Fix flake8 errors --- qutebrowser/browser/history.py | 2 +- tests/unit/misc/test_sql.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e314f6c5e..6df09ce16 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -158,7 +158,7 @@ class WebHistory(sql.SqlTable): self.completion.insert({'url': url_str, 'title': title, 'last_atime': atime}, - replace=True) + replace=True) def _parse_entry(self, line): """Parse a history line like '12345 http://example.com title'.""" diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index afd952255..1f67f0bee 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -79,7 +79,7 @@ def test_insert_batch_replace(qtbot): table.insert_batch({'name': ['one', 'nine'], 'val': [11, 19], 'lucky': [True, True]}, - replace=True) + replace=True) assert list(table) == [('thirteen', 13, True), ('one', 11, True), From 051d2665f3ec4d83904e29fe4f5c80befef9d725 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 14 Jun 2017 07:17:30 -0400 Subject: [PATCH 120/161] Fix signal type error in CompletionView. On Travis CI we are sometimes seeing: ``` CompletionView.selection_changed[str].emit(): argument 1 has unexpected type 'int' ``` Cast the data to a string before emitting it just to be safe. --- qutebrowser/completion/completionwidget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 7ea4edf13..60ecd5b88 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -344,7 +344,7 @@ class CompletionView(QTreeView): indexes = selected.indexes() if not indexes: return - data = self.model().data(indexes[0]) + data = str(self.model().data(indexes[0])) self.selection_changed.emit(data) def resizeEvent(self, e): From 9f94f28181333c8023ea96cfd10d67e60dc70491 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 19 Jun 2017 09:14:03 +0200 Subject: [PATCH 121/161] Use :memory: for an in-memory database Using an empty string for the same purpose only started working in some recent-ish Qt/sqlite/? version, so using --version failed on Ubuntu Trusty. --- qutebrowser/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index a8fd7ad54..4eefb38d8 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -85,7 +85,7 @@ def run(args): if args.version: # we need to init sql to print the sql version # we can use an in-memory database as we just want to query the version - sql.init('') + sql.init(':memory:') print(version.version()) sys.exit(usertypes.Exit.ok) From 4296a61b9a6cc562380c2e333bd9c3d33d772f50 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 19 Jun 2017 09:19:02 +0200 Subject: [PATCH 122/161] tests: Clean up check_history --- tests/end2end/features/test_history_bdd.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index f5639f5f3..319e36aee 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -32,10 +32,9 @@ def check_history(quteproc, httpbin, tmpdir, expected): quteproc.wait_for(category='message', loglevel=logging.INFO, message='Dumped history to {}'.format(path)) - with open(path, 'r', encoding='utf-8') as f: + with path.open('r', encoding='utf-8') as f: # ignore access times, they will differ in each run - actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() - for line in f.read().splitlines()) + actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() for line in f) expected = expected.replace('(port)', str(httpbin.port)) assert actual == expected From 29ce1b381121645589cb18906682fe79b11a69b8 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 20 Jun 2017 11:56:17 +0200 Subject: [PATCH 123/161] Add column order in SqlTable._insert_query _insert_query gets called with a query and dict of values such as: {'val': 1, 'lucky': False, 'name': 'one'} Via bindValues(), we only assign a placeholder in the query string to a value, so we get a query with bindings like: INSERT INTO Foo values(:lucky,:val,:name) {':name': 'one', ':val': 1, ':lucky': False} So what we're executing is something like: INSERT INTO Foo values(false,1,"one") However, if the column order in the database doesn't happen to be the order we're passing the values in, we get the wrong values in the wrong columns. Instead, we now do: INSERT INTO Foo (lucky, val, name) values(false,1,"one") Which inserts the values in the order we intended. With Python 3.6, this just happened to work before because we always passed the keyword arguments in the table column order, and in 3.6 dicts (and thus **kwargs) happen to be ordered: https://mail.python.org/pipermail/python-dev/2016-September/146327.html --- qutebrowser/misc/sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 717ae3c0f..1f93f8c06 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -187,7 +187,8 @@ class SqlTable(QObject): def _insert_query(self, values, replace): params = ','.join(':{}'.format(key) for key in values) verb = "REPLACE" if replace else "INSERT" - return Query("{} INTO {} values({})".format(verb, self._name, params)) + return Query("{} INTO {} ({}) values({})".format( + verb, self._name, ','.join(values), params)) def insert(self, values, replace=False): """Append a row to the table. From c1776bbf9d00bb4c6bfe0c5bfc03c460e4eae411 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 20 Jun 2017 12:03:31 +0200 Subject: [PATCH 124/161] Add error message when query failed to prepare --- qutebrowser/misc/sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 1f93f8c06..3bd02352c 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -70,7 +70,8 @@ class Query(QSqlQuery): super().__init__(QSqlDatabase.database()) log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) if not self.prepare(querystr): - raise SqlException('Failed to prepare query "{}"'.format(querystr)) + raise SqlException('Failed to prepare query "{}": "{}"'.format( + querystr, self.lastError().text())) self.setForwardOnly(forward_only) def __iter__(self): From da875755d112f677933e14255c986fef10cf6809 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 20 Jun 2017 12:06:17 +0200 Subject: [PATCH 125/161] Add spaces after commas in SQL queries --- qutebrowser/misc/sql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 3bd02352c..585085cc1 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -133,7 +133,7 @@ class SqlTable(QObject): column_defs = ['{} {}'.format(field, constraints.get(field, '')) for field in fields] q = Query("CREATE TABLE IF NOT EXISTS {} ({})" - .format(name, ','.join(column_defs))) + .format(name, ', '.join(column_defs))) q.run() @@ -186,10 +186,10 @@ class SqlTable(QObject): self.changed.emit() def _insert_query(self, values, replace): - params = ','.join(':{}'.format(key) for key in values) + params = ', '.join(':{}'.format(key) for key in values) verb = "REPLACE" if replace else "INSERT" return Query("{} INTO {} ({}) values({})".format( - verb, self._name, ','.join(values), params)) + verb, self._name, ', '.join(values), params)) def insert(self, values, replace=False): """Append a row to the table. From f838eb1bdc763d4f9b5b46f4778ccd56f05348d6 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 20 Jun 2017 12:11:02 +0200 Subject: [PATCH 126/161] Use named formatting for queries in sql.py --- qutebrowser/misc/sql.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 585085cc1..987fac59c 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -132,8 +132,8 @@ class SqlTable(QObject): constraints = constraints or {} column_defs = ['{} {}'.format(field, constraints.get(field, '')) for field in fields] - q = Query("CREATE TABLE IF NOT EXISTS {} ({})" - .format(name, ', '.join(column_defs))) + q = Query("CREATE TABLE IF NOT EXISTS {name} ({column_defs})" + .format(name=name, column_defs=', '.join(column_defs))) q.run() @@ -144,13 +144,13 @@ class SqlTable(QObject): name: Name of the index, should be unique. field: Name of the field to index. """ - q = Query("CREATE INDEX IF NOT EXISTS {} ON {} ({})" - .format(name, self._name, field)) + q = Query("CREATE INDEX IF NOT EXISTS {name} ON {table} ({field})" + .format(name=name, table=self._name, field=field)) q.run() def __iter__(self): """Iterate rows in the table.""" - q = Query("SELECT * FROM {}".format(self._name)) + q = Query("SELECT * FROM {table}".format(table=self._name)) q.run() return iter(q) @@ -160,12 +160,13 @@ class SqlTable(QObject): Args: field: Field to match. """ - return Query("SELECT EXISTS(SELECT * FROM {} WHERE {} = :val)" - .format(self._name, field)) + return Query( + "SELECT EXISTS(SELECT * FROM {table} WHERE {field} = :val)" + .format(table=self._name, field=field)) def __len__(self): """Return the count of rows in the table.""" - q = Query("SELECT count(*) FROM {}".format(self._name)) + q = Query("SELECT count(*) FROM {table}".format(table=self._name)) q.run() return q.value() @@ -179,7 +180,8 @@ class SqlTable(QObject): Return: The number of rows deleted. """ - q = Query("DELETE FROM {} where {} = :val".format(self._name, field)) + q = Query("DELETE FROM {table} where {field} = :val" + .format(table=self._name, field=field)) q.run(val=value) if not q.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) @@ -188,8 +190,9 @@ class SqlTable(QObject): def _insert_query(self, values, replace): params = ', '.join(':{}'.format(key) for key in values) verb = "REPLACE" if replace else "INSERT" - return Query("{} INTO {} ({}) values({})".format( - verb, self._name, ', '.join(values), params)) + return Query("{verb} INTO {table} ({columns}) values({params})".format( + verb=verb, table=self._name, columns=', '.join(values), + params=params)) def insert(self, values, replace=False): """Append a row to the table. @@ -223,7 +226,7 @@ class SqlTable(QObject): def delete_all(self): """Remove all rows from the table.""" - Query("DELETE FROM {}".format(self._name)).run() + Query("DELETE FROM {table}".format(table=self._name)).run() self.changed.emit() def select(self, sort_by, sort_order, limit=-1): @@ -236,7 +239,8 @@ class SqlTable(QObject): Return: A prepared and executed select query. """ - q = Query('SELECT * FROM {} ORDER BY {} {} LIMIT :limit' - .format(self._name, sort_by, sort_order)) + q = Query("SELECT * FROM {table} ORDER BY {sort_by} {sort_order} " + "LIMIT :limit" + .format(table=self._name, sort_by=sort_by, sort_order=sort_order)) q.run(limit=limit) return q From 038dcff4ba5f76444a4ac6880762f51d8b190414 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 20 Jun 2017 12:16:03 +0200 Subject: [PATCH 127/161] Ignore common URL issues while importing history See #2646. This ignores the "corrupted" Apple-lookalike URLs, comments and data: URLs. --- qutebrowser/browser/history.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 6df09ce16..58961f232 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -171,10 +171,19 @@ class WebHistory(sql.SqlTable): else: raise ValueError("2 or 3 fields expected") + # http://xn--pple-43d.com/ with + # https://bugreports.qt.io/browse/QTBUG-60364 + if url in ['http://.com/', 'https://www..com/']: + return None + url = QUrl(url) if not url.isValid(): raise ValueError("Invalid URL: {}".format(url.errorString())) + # https://github.com/qutebrowser/qutebrowser/issues/2646 + if url.scheme() == 'data': + return None + # https://github.com/qutebrowser/qutebrowser/issues/670 atime = atime.lstrip('\0') @@ -223,10 +232,13 @@ class WebHistory(sql.SqlTable): completion_data = {'url': [], 'title': [], 'last_atime': []} for (i, line) in enumerate(f): line = line.strip() - if not line: + if not line or line.startswith('#'): continue try: - url, title, atime, redirect = self._parse_entry(line) + parsed = self._parse_entry(line) + if parsed is None: + continue + url, title, atime, redirect = parsed data['url'].append(url) data['title'].append(title) data['atime'].append(atime) From 0f585eda4f5d958eada969e75bf412601a904f1e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Jun 2017 21:41:43 -0400 Subject: [PATCH 128/161] Bring history.py back to 100% coverage. The code of debug_dump_history was tweaked to handle a possible OSException that can be thrown by open, which I noticed while trying to test it. --- qutebrowser/browser/history.py | 10 ++--- tests/unit/browser/webkit/test_history.py | 54 ++++++++++++++++++++++- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 58961f232..67d6b6994 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -270,12 +270,12 @@ class WebHistory(sql.SqlTable): .format(int(x.atime), '-r' * x.redirect, x.url, x.title) for x in self.select(sort_by='atime', sort_order='asc')) - with open(dest, 'w', encoding='utf-8') as f: - try: + try: + with open(dest, 'w', encoding='utf-8') as f: f.write('\n'.join(lines)) - except OSError as e: - raise cmdexc.CommandError('Could not write history: {}', e) - message.info("Dumped history to {}".format(dest)) + message.info("Dumped history to {}".format(dest)) + except OSError as e: + raise cmdexc.CommandError('Could not write history: {}', e) def init(parent=None): diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 04081e7c7..01178083a 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -20,12 +20,14 @@ """Tests for the global page history.""" import logging +import os import pytest from PyQt5.QtCore import QUrl from qutebrowser.browser import history from qutebrowser.utils import objreg, urlutils, usertypes +from qutebrowser.commands import cmdexc @pytest.fixture(autouse=True) @@ -144,6 +146,8 @@ def test_add_item_invalid(qtbot, hist, caplog): ('b.com', 'title', 12345, True)]), (logging.WARNING, 'a.com', '', [('a.com', 'title', 12345, False)]), (logging.WARNING, '', '', []), + (logging.WARNING, 'data:foo', '', []), + (logging.WARNING, 'a.com', 'data:foo', []), ]) def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog): with caplog.at_level(level): @@ -219,7 +223,7 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): assert default_interface is None -def test_read(hist, data_tmpdir, monkeypatch, stubs): +def test_import_txt(hist, data_tmpdir, monkeypatch, stubs): monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) histfile = data_tmpdir / 'history' # empty line is deliberate, to test skipping empty lines @@ -242,13 +246,39 @@ def test_read(hist, data_tmpdir, monkeypatch, stubs): assert (data_tmpdir / 'history.bak').exists() +@pytest.mark.parametrize('line', [ + '', + '#12345 http://example.com/commented', + + # https://bugreports.qt.io/browse/QTBUG-60364 + '12345 http://.com/', + '12345 https://www..com/', + + # issue #2646 + '12345 data:text/html;charset=UTF-8,%3C%21DOCTYPE%20html%20PUBLIC%20%22-', +]) +def test_import_txt_skip(hist, data_tmpdir, line, monkeypatch, stubs): + """import_txt should skip certain lines silently.""" + monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) + histfile = data_tmpdir / 'history' + histfile.write(line) + + hist.import_txt() + + assert not histfile.exists() + assert not len(hist) + + @pytest.mark.parametrize('line', [ 'xyz http://example.com/bad-timestamp', '12345', 'http://example.com/no-timestamp', '68891-r-r http://example.com/double-flag', + '68891-x http://example.com/bad-flag', + '68891 http://.com', ]) -def test_read_invalid(hist, data_tmpdir, line, monkeypatch, stubs): +def test_import_txt_invalid(hist, data_tmpdir, line, monkeypatch, stubs): + """import_txt should fail on certain lines.""" monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) histfile = data_tmpdir / 'history' histfile.write(line) @@ -259,6 +289,12 @@ def test_read_invalid(hist, data_tmpdir, line, monkeypatch, stubs): assert histfile.exists() +def test_import_txt_nonexistant(hist, data_tmpdir, monkeypatch, stubs): + """import_txt should do nothing if the history file doesn't exist.""" + monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) + hist.import_txt() + + def test_debug_dump_history(hist, tmpdir): hist.add_url(QUrl('http://example.com/1'), title="Title1", atime=12345) hist.add_url(QUrl('http://example.com/2'), title="Title2", atime=12346) @@ -272,3 +308,17 @@ def test_debug_dump_history(hist, tmpdir): '12347 http://example.com/3 Title3', '12348-r http://example.com/4 Title4'] assert histfile.read() == '\n'.join(expected) + + +def test_debug_dump_history_nonexistant(hist, tmpdir): + histfile = tmpdir / 'nonexistant' / 'history' + with pytest.raises(cmdexc.CommandError): + hist.debug_dump_history(str(histfile)) + + +def test_debug_dump_history_oserror(hist, tmpdir): + histfile = tmpdir / 'history' + histfile.write('') + os.chmod(str(histfile), 0) + with pytest.raises(cmdexc.CommandError): + hist.debug_dump_history(str(histfile)) From 63cb88a0f4eb5536b89906ae0120bc472cdc76a9 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Jun 2017 22:08:23 -0400 Subject: [PATCH 129/161] Use _cat_from_index in completionmodel.data. Keep all the category lookup inside _cat_from_idx for easier refactoring if the organization ever changes. --- qutebrowser/completion/models/completionmodel.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 2d74d94e0..62148e9d3 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -75,15 +75,18 @@ class CompletionModel(QAbstractItemModel): Return: The item data, or None on an invalid index. """ - if not index.isValid() or role != Qt.DisplayRole: + if role != Qt.DisplayRole: return None - if not index.parent().isValid(): + cat = self._cat_from_idx(index) + if cat: # category header if index.column() == 0: return self._categories[index.row()].name return None # item - cat = self._categories[index.parent().row()] + cat = self._cat_from_idx(index.parent()) + if not cat: + return None idx = cat.index(index.row(), index.column()) return cat.data(idx) From b722cc1dec6ce86e8f1499879dcc4859430b09c5 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Jun 2017 22:11:53 -0400 Subject: [PATCH 130/161] Pass invalid index to [can]FetchMore. For QSqlQueryModel, the argument should always be an invalid index: http://doc.qt.io/qt-5/qsqlquerymodel.html#canFetchMore For a QStandardItemModel, it doesn't matter. Either way, passing the top-level parent index was wrong. --- qutebrowser/completion/models/completionmodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 62148e9d3..deef088cf 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -161,14 +161,14 @@ class CompletionModel(QAbstractItemModel): """Override to forward the call to the categories.""" cat = self._cat_from_idx(parent) if cat: - return cat.canFetchMore(parent) + return cat.canFetchMore(QModelIndex()) return False def fetchMore(self, parent): """Override to forward the call to the categories.""" cat = self._cat_from_idx(parent) if cat: - cat.fetchMore(parent) + cat.fetchMore(QModelIndex()) def count(self): """Return the count of non-category items.""" From 6080830a8b1cc27f5a14d68637c16e325e1fdf9c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Jun 2017 22:25:09 -0400 Subject: [PATCH 131/161] Fix outdated docstring and pylint error. --- qutebrowser/completion/models/listcategory.py | 6 +----- qutebrowser/misc/sql.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index acdc5cafc..ce10bbb14 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -17,11 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""The base completion model for completion in the command line. - -Module attributes: - Role: An enum of user defined model roles. -""" +"""Completion category that uses a list of tuples as a data source.""" import re diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 987fac59c..52ae13e11 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -241,6 +241,7 @@ class SqlTable(QObject): """ q = Query("SELECT * FROM {table} ORDER BY {sort_by} {sort_order} " "LIMIT :limit" - .format(table=self._name, sort_by=sort_by, sort_order=sort_order)) + .format(table=self._name, sort_by=sort_by, + sort_order=sort_order)) q.run(limit=limit) return q From 866f4653c77d0477f3df00af518f21565e07d8b9 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 25 Jun 2017 22:14:01 -0400 Subject: [PATCH 132/161] Fix spelling existant -> existent. --- tests/unit/browser/webkit/test_history.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 01178083a..21945b537 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -289,7 +289,7 @@ def test_import_txt_invalid(hist, data_tmpdir, line, monkeypatch, stubs): assert histfile.exists() -def test_import_txt_nonexistant(hist, data_tmpdir, monkeypatch, stubs): +def test_import_txt_nonexistent(hist, data_tmpdir, monkeypatch, stubs): """import_txt should do nothing if the history file doesn't exist.""" monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) hist.import_txt() @@ -310,8 +310,8 @@ def test_debug_dump_history(hist, tmpdir): assert histfile.read() == '\n'.join(expected) -def test_debug_dump_history_nonexistant(hist, tmpdir): - histfile = tmpdir / 'nonexistant' / 'history' +def test_debug_dump_history_nonexistent(hist, tmpdir): + histfile = tmpdir / 'nonexistent' / 'history' with pytest.raises(cmdexc.CommandError): hist.debug_dump_history(str(histfile)) From 46161c3af0189ca2616e9bff0ac626bf2ea047f3 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Jun 2017 23:04:03 -0400 Subject: [PATCH 133/161] Refactor delete_cur_item. Taking the completion widget as an argument was overly complex. The process now looks like: 1. CompletionView gets deletion request 2. CompletionView passes selected index to CompletionModel 3. CompletionModel passes the row data to the owning category 4. The category runs its custom completion function. This also fixes a bug. With the switch to the hybrid (list/sql) completion model, the view was no longer updating when items were deleted. This fixes that by ensuring the correct signals are emitted. The SQL model must be refreshed by running the query. We could try using a SqlTableModel so we can call removeRows instead. The test for deleting a url fails because qmodeltester claims the length of the query model is still 3. --- qutebrowser/browser/history.py | 9 ++ qutebrowser/completion/completionwidget.py | 8 +- .../completion/models/completionmodel.py | 15 ++- qutebrowser/completion/models/listcategory.py | 16 ++- qutebrowser/completion/models/miscmodels.py | 18 +-- qutebrowser/completion/models/sqlcategory.py | 17 ++- qutebrowser/completion/models/urlmodel.py | 57 ++++----- tests/helpers/fixtures.py | 9 ++ tests/helpers/stubs.py | 29 +++++ tests/unit/completion/test_completionmodel.py | 23 +++- .../unit/completion/test_completionwidget.py | 26 ++-- tests/unit/completion/test_models.py | 120 ++++++++++-------- 12 files changed, 220 insertions(+), 127 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 67d6b6994..e9a6ce980 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -116,6 +116,15 @@ class WebHistory(sql.SqlTable): def _do_clear(self): self.delete_all() + def delete_url(self, url): + """Remove all history entries with the given url. + + Args: + url: URL string to delete. + """ + self.delete(url, 'url') + self.completion.delete(url, 'url') + @pyqtSlot(QUrl, QUrl, str) def add_from_tab(self, url, requested_url, title): """Add a new history entry as slot, called from a BrowserTab.""" diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 60ecd5b88..415968131 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -364,9 +364,7 @@ class CompletionView(QTreeView): modes=[usertypes.KeyMode.command], scope='window') def completion_item_del(self): """Delete the current completion item.""" - if not self.currentIndex().isValid(): + index = self.currentIndex() + if not index.isValid(): raise cmdexc.CommandError("No item selected!") - if self.model().delete_cur_item is None: - raise cmdexc.CommandError("Cannot delete this item.") - else: - self.model().delete_cur_item(self) + self.model().delete_cur_item(index) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index deef088cf..55b0546d9 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -22,6 +22,7 @@ from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from qutebrowser.utils import log, qtutils +from qutebrowser.commands import cmdexc class CompletionModel(QAbstractItemModel): @@ -38,13 +39,11 @@ class CompletionModel(QAbstractItemModel): _categories: The sub-categories. """ - def __init__(self, *, column_widths=(30, 70, 0), - delete_cur_item=None, parent=None): + def __init__(self, *, column_widths=(30, 70, 0), parent=None): super().__init__(parent) self.column_widths = column_widths self._categories = [] self.pattern = '' - self.delete_cur_item = delete_cur_item def _cat_from_idx(self, index): """Return the category pointed to by the given index. @@ -217,3 +216,13 @@ class CompletionModel(QAbstractItemModel): """ cat = self._cat_from_idx(index.parent()) return cat.columns_to_filter if cat else [] + + def delete_cur_item(self, index): + """Delete the row at the given index.""" + parent = index.parent() + cat = self._cat_from_idx(parent) + if not cat: + raise cmdexc.CommandError("No category selected") + self.beginRemoveRows(parent, index.row(), index.row()) + cat.delete_cur_item(cat.index(index.row(), 0)) + self.endRemoveRows() diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index ce10bbb14..c02783e66 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -21,17 +21,19 @@ import re -from PyQt5.QtCore import QSortFilterProxyModel +from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex from PyQt5.QtGui import QStandardItem, QStandardItemModel from qutebrowser.utils import qtutils +from qutebrowser.commands import cmdexc class ListCategory(QSortFilterProxyModel): """Expose a list of items as a category for the CompletionModel.""" - def __init__(self, name, items, columns_to_filter=None, parent=None): + def __init__(self, name, items, columns_to_filter=None, + delete_func=None, parent=None): super().__init__(parent) self.name = name self.srcmodel = QStandardItemModel(parent=self) @@ -41,6 +43,7 @@ class ListCategory(QSortFilterProxyModel): for item in items: self.srcmodel.appendRow([QStandardItem(x) for x in item]) self.setSourceModel(self.srcmodel) + self.delete_func = delete_func def set_pattern(self, val): """Setter for pattern. @@ -114,3 +117,12 @@ class ListCategory(QSortFilterProxyModel): return False else: return left < right + + def delete_cur_item(self, index): + """Delete the row at the given index.""" + if not self.delete_func: + raise cmdexc.CommandError("Cannot delete this item.") + data = [self.data(index.sibling(index.row(), i)) + for i in range(self.columnCount())] + self.delete_func(data) + self.removeRow(index.row(), QModelIndex()) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 62818f767..8338452c9 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -20,7 +20,7 @@ """Functions that return miscellaneous completion models.""" from qutebrowser.config import config, configdata -from qutebrowser.utils import objreg, log, qtutils +from qutebrowser.utils import objreg, log from qutebrowser.commands import cmdutils from qutebrowser.completion.models import completionmodel, listcategory @@ -96,21 +96,14 @@ def buffer(): url_column = 1 text_column = 2 - def delete_buffer(completion): + def delete_buffer(data): """Close the selected tab.""" - index = completion.currentIndex() - qtutils.ensure_valid(index) - category = index.parent() - qtutils.ensure_valid(category) - index = category.child(index.row(), idx_column) - win_id, tab_index = index.data().split('/') + win_id, tab_index = data[0].split('/') tabbed_browser = objreg.get('tabbed-browser', scope='window', window=int(win_id)) tabbed_browser.on_tab_close_requested(int(tab_index) - 1) - model = completionmodel.CompletionModel( - column_widths=(6, 40, 54), - delete_cur_item=delete_buffer) + model = completionmodel.CompletionModel(column_widths=(6, 40, 54)) for win_id in objreg.window_registry: tabbed_browser = objreg.get('tabbed-browser', scope='window', @@ -124,7 +117,8 @@ def buffer(): tab.url().toDisplayString(), tabbed_browser.page_title(idx))) cat = listcategory.ListCategory("{}".format(win_id), tabs, - columns_to_filter=[idx_column, url_column, text_column]) + columns_to_filter=[idx_column, url_column, text_column], + delete_func=delete_buffer) model.add_category(cat) return model diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 9e88b52da..9edd5f96e 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -21,10 +21,12 @@ import re +from PyQt5.QtCore import QModelIndex from PyQt5.QtSql import QSqlQueryModel from qutebrowser.misc import sql from qutebrowser.utils import debug +from qutebrowser.commands import cmdexc class SqlCategory(QSqlQueryModel): @@ -33,7 +35,7 @@ class SqlCategory(QSqlQueryModel): def __init__(self, name, *, title=None, filter_fields, sort_by=None, sort_order=None, select='*', where=None, group_by=None, - parent=None): + delete_func=None, parent=None): """Create a new completion category backed by a sql table. Args: @@ -44,6 +46,7 @@ class SqlCategory(QSqlQueryModel): where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Either 'asc' or 'desc', if sort_by is non-None + delete_func: Callback to delete a selected item. """ super().__init__(parent=parent) self.name = title or name @@ -69,6 +72,7 @@ class SqlCategory(QSqlQueryModel): col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name)) rec = col_query.run().record() self.columns_to_filter = [rec.indexOf(n) for n in filter_fields] + self.delete_func = delete_func def set_pattern(self, pattern): """Set the pattern used to filter results. @@ -86,3 +90,14 @@ class SqlCategory(QSqlQueryModel): with debug.log_time('sql', 'Running completion query'): self._query.run(pattern=pattern) self.setQuery(self._query) + + def delete_cur_item(self, index): + """Delete the row at the given index.""" + if not self.delete_func: + raise cmdexc.CommandError("Cannot delete this item.") + data = [self.data(index.sibling(index.row(), i)) + for i in range(self.columnCount())] + self.delete_func(data) + # re-run query to reload updated table + with debug.log_time('sql', 'Running completion query'): + self._query.run() diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 8cacca873..397783d6f 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -29,31 +29,25 @@ _URLCOL = 0 _TEXTCOL = 1 -def _delete_url(completion): - """Delete the selected item. +def _delete_history(data): + urlstr = data[_URLCOL] + log.completion.debug('Deleting history entry {}'.format(urlstr)) + hist = objreg.get('web-history') + hist.delete_url(urlstr) - Args: - completion: The Completion object to use. - """ - index = completion.currentIndex() - qtutils.ensure_valid(index) - category = index.parent() - index = category.child(index.row(), _URLCOL) - catname = category.data() - qtutils.ensure_valid(category) - if catname == 'Bookmarks': - urlstr = index.data() - log.completion.debug('Deleting bookmark {}'.format(urlstr)) - bookmark_manager = objreg.get('bookmark-manager') - bookmark_manager.delete(urlstr) - elif catname == 'Quickmarks': - quickmark_manager = objreg.get('quickmark-manager') - sibling = index.sibling(index.row(), _TEXTCOL) - qtutils.ensure_valid(sibling) - name = sibling.data() - log.completion.debug('Deleting quickmark {}'.format(name)) - quickmark_manager.delete(name) +def _delete_bookmark(data): + urlstr = data[_URLCOL] + log.completion.debug('Deleting bookmark {}'.format(urlstr)) + bookmark_manager = objreg.get('bookmark-manager') + bookmark_manager.delete(urlstr) + + +def _delete_quickmark(data): + name = data[_TEXTCOL] + quickmark_manager = objreg.get('quickmark-manager') + log.completion.debug('Deleting quickmark {}'.format(name)) + quickmark_manager.delete(name) def url(): @@ -61,18 +55,18 @@ def url(): Used for the `open` command. """ - model = completionmodel.CompletionModel( - column_widths=(40, 50, 10), - delete_cur_item=_delete_url) + model = completionmodel.CompletionModel(column_widths=(40, 50, 10)) quickmarks = ((url, name) for (name, url) in objreg.get('quickmark-manager').marks.items()) bookmarks = objreg.get('bookmark-manager').marks.items() - model.add_category(listcategory.ListCategory('Quickmarks', quickmarks, - columns_to_filter=[0, 1])) - model.add_category(listcategory.ListCategory('Bookmarks', bookmarks, - columns_to_filter=[0, 1])) + model.add_category(listcategory.ListCategory( + 'Quickmarks', quickmarks, columns_to_filter=[0, 1], + delete_func=_delete_quickmark)) + model.add_category(listcategory.ListCategory( + 'Bookmarks', bookmarks, columns_to_filter=[0, 1], + delete_func=_delete_bookmark)) # replace 's to avoid breaking the query timefmt = config.get('completion', 'timestamp-format').replace("'", "`") @@ -81,6 +75,7 @@ def url(): 'CompletionHistory', title='History', sort_order='desc', sort_by='last_atime', filter_fields=['url', 'title'], - select='url, title, {}'.format(select_time)) + select='url, title, {}'.format(select_time), + delete_func=_delete_history) model.add_category(hist_cat) return model diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 1a3dfd8ea..4fd6a0731 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -257,6 +257,15 @@ def bookmark_manager_stub(stubs): objreg.delete('bookmark-manager') +@pytest.fixture +def web_history_stub(init_sql, stubs): + """Fixture which provides a fake web-history object.""" + stub = stubs.WebHistoryStub() + objreg.register('web-history', stub) + yield stub + objreg.delete('web-history') + + @pytest.fixture def session_manager_stub(stubs): """Fixture which provides a fake session-manager object.""" diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 717ce4c18..a490ad5bb 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -33,6 +33,7 @@ from qutebrowser.browser import browsertab from qutebrowser.config import configexc from qutebrowser.utils import usertypes, utils from qutebrowser.mainwindow import mainwindow +from qutebrowser.misc import sql class FakeNetworkCache(QAbstractNetworkCache): @@ -522,6 +523,34 @@ class QuickmarkManagerStub(UrlMarkManagerStub): self.delete(key) +class WebHistoryStub(sql.SqlTable): + + """Stub for the web-history object.""" + + def __init__(self): + super().__init__("History", ['url', 'title', 'atime', 'redirect']) + self.completion = sql.SqlTable("CompletionHistory", + ['url', 'title', 'last_atime']) + + def add_url(self, url, title="", *, redirect=False, atime=None): + self.insert({'url': url, 'title': title, 'atime': atime, + 'redirect': redirect}) + if not redirect: + self.completion.insert({'url': url, + 'title': title, + 'last_atime': atime}) + + + def delete_url(self, url): + """Remove all history entries with the given url. + + Args: + url: URL string to delete. + """ + self.delete(url, 'url') + self.completion.delete(url, 'url') + + class HostBlockerStub: """Stub for the host-blocker object.""" diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 8f8acced2..5f4670f97 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -23,7 +23,11 @@ from unittest import mock import hypothesis from hypothesis import strategies -from qutebrowser.completion.models import completionmodel +import pytest +from PyQt5.QtCore import QModelIndex + +from qutebrowser.completion.models import completionmodel, listcategory +from qutebrowser.commands import cmdexc @hypothesis.given(strategies.lists(min_size=0, max_size=3, @@ -72,3 +76,20 @@ def test_set_pattern(pat): model.add_category(c) model.set_pattern(pat) assert all(c.set_pattern.called_with([pat]) for c in cats) + + +def test_delete_cur_item(): + func = mock.Mock() + model = completionmodel.CompletionModel() + cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=func) + model.add_category(cat) + parent = model.index(0, 0) + model.delete_cur_item(model.index(0, 0, parent)) + func.assert_called_once_with(['foo', 'bar']) + + +def test_delete_cur_item_no_cat(): + """Test completion_item_del with no selected category.""" + model = completionmodel.CompletionModel() + with pytest.raises(cmdexc.CommandError, match='No category selected'): + model.delete_cur_item(QModelIndex()) diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 4cc59deea..03f352194 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -215,30 +215,22 @@ def test_completion_show(show, rows, quick_complete, completionview, def test_completion_item_del(completionview): """Test that completion_item_del invokes delete_cur_item in the model.""" func = mock.Mock() - model = completionmodel.CompletionModel(delete_cur_item=func) - model.add_category(listcategory.ListCategory('', [('foo',)])) + model = completionmodel.CompletionModel() + cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=func) + model.add_category(cat) completionview.set_model(model) completionview.completion_item_focus('next') completionview.completion_item_del() - assert func.called + func.assert_called_once_with(['foo', 'bar']) def test_completion_item_del_no_selection(completionview): - """Test that completion_item_del with no selected index.""" + """Test that completion_item_del with an invalid index.""" func = mock.Mock() - model = completionmodel.CompletionModel(delete_cur_item=func) - model.add_category(listcategory.ListCategory('', [('foo',)])) + model = completionmodel.CompletionModel() + cat = listcategory.ListCategory('', [('foo',)], delete_func=func) + model.add_category(cat) completionview.set_model(model) with pytest.raises(cmdexc.CommandError, match='No item selected!'): completionview.completion_item_del() - assert not func.called - - -def test_completion_item_del_no_func(completionview): - """Test completion_item_del with no delete_cur_item in the model.""" - model = completionmodel.CompletionModel() - model.add_category(listcategory.ListCategory('', [('foo',)])) - completionview.set_model(model) - completionview.completion_item_focus('next') - with pytest.raises(cmdexc.CommandError, match='Cannot delete this item.'): - completionview.completion_item_del() + func.assert_not_called diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index e9e8e6366..5677eab52 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -20,6 +20,7 @@ """Tests for completion models.""" import collections +import unittest.mock from datetime import datetime import pytest @@ -28,6 +29,7 @@ from PyQt5.QtWidgets import QTreeView from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value +from qutebrowser.utils import objreg from qutebrowser.misc import sql @@ -114,23 +116,6 @@ def _patch_config_section_desc(monkeypatch, stubs, symbol): monkeypatch.setattr(symbol, section_desc) -def _mock_view_index(model, category_num, child_num, qtbot): - """Create a tree view from a model and set the current index. - - Args: - model: model to create a fake view for. - category_idx: index of the category to select. - child_idx: index of the child item under that category to select. - """ - view = QTreeView() - qtbot.add_widget(view) - view.setModel(model) - parent = model.index(category_num, 0) - child = model.index(child_num, 0, parent=parent) - view.setCurrentIndex(child) - return view - - @pytest.fixture def quickmarks(quickmark_manager_stub): """Pre-populate the quickmark-manager stub with some quickmarks.""" @@ -152,24 +137,29 @@ def bookmarks(bookmark_manager_stub): ]) return bookmark_manager_stub - @pytest.fixture -def web_history_stub(stubs, init_sql): +def history_completion_table(init_sql): return sql.SqlTable("CompletionHistory", ['url', 'title', 'last_atime']) @pytest.fixture -def web_history(web_history_stub, init_sql): +def web_history(web_history_stub): """Pre-populate the web-history database.""" - web_history_stub.insert({'url': 'http://qutebrowser.org', - 'title': 'qutebrowser', - 'last_atime': datetime(2015, 9, 5).timestamp()}) - web_history_stub.insert({'url': 'https://python.org', - 'title': 'Welcome to Python.org', - 'last_atime': datetime(2016, 3, 8).timestamp()}) - web_history_stub.insert({'url': 'https://github.com', - 'title': 'https://github.com', - 'last_atime': datetime(2016, 5, 1).timestamp()}) + web_history_stub.add_url( + url='http://qutebrowser.org', + title='qutebrowser', + atime=datetime(2015, 9, 5).timestamp() + ) + web_history_stub.add_url( + url='https://python.org', + title='Welcome to Python.org', + atime=datetime(2016, 3, 8).timestamp() + ) + web_history_stub.add_url( + url='https://github.com', + title='https://github.com', + atime=datetime(2016, 5, 1).timestamp() + ) return web_history_stub @@ -280,7 +270,6 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, Verify that: - quickmarks, bookmarks, and urls are included - entries are sorted by access time - - redirect entries are not included - only the most recent entry is included for each url """ config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} @@ -331,16 +320,15 @@ def test_url_completion_pattern(config_stub, web_history_stub, url, title, pattern, rowcount): """Test that url completion filters by url and title.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} - web_history_stub.insert({'url': url, 'title': title, 'last_atime': 0}) + web_history_stub.add_url(url, title) model = urlmodel.url() model.set_pattern(pattern) # 2, 0 is History assert model.rowCount(model.index(2, 0)) == rowcount -def test_url_completion_delete_bookmark(qtmodeltester, config_stub, - web_history, quickmarks, bookmarks, - qtbot): +def test_url_completion_delete_bookmark(qtmodeltester, config_stub, bookmarks, + web_history, quickmarks): """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() @@ -348,12 +336,18 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - # delete item (1, 1) -> (bookmarks, 'https://github.com') - view = _mock_view_index(model, 1, 1, qtbot) - model.delete_cur_item(view) + parent = model.index(1, 0) + idx = model.index(1, 0, parent) + + # sanity checks + assert model.data(parent) == "Bookmarks" + assert model.data(idx) == 'https://github.com' + assert 'https://github.com' in bookmarks.marks + + len_before = len(bookmarks.marks) + model.delete_cur_item(idx) assert 'https://github.com' not in bookmarks.marks - assert 'https://python.org' in bookmarks.marks - assert 'http://qutebrowser.org' in bookmarks.marks + assert len_before == len(bookmarks.marks) + 1 def test_url_completion_delete_quickmark(qtmodeltester, config_stub, @@ -366,28 +360,39 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - # delete item (0, 0) -> (quickmarks, 'ddg' ) - view = _mock_view_index(model, 0, 0, qtbot) - model.delete_cur_item(view) - assert 'aw' in quickmarks.marks + parent = model.index(0, 0) + idx = model.index(0, 0, parent) + + # sanity checks + assert model.data(parent) == "Quickmarks" + assert model.data(idx) == 'https://duckduckgo.com' + assert 'ddg' in quickmarks.marks + + len_before = len(quickmarks.marks) + model.delete_cur_item(idx) assert 'ddg' not in quickmarks.marks - assert 'wiki' in quickmarks.marks + assert len_before == len(quickmarks.marks) + 1 def test_url_completion_delete_history(qtmodeltester, config_stub, - web_history, quickmarks, bookmarks, - qtbot): - """Test that deleting a history entry is a noop.""" + web_history_stub, web_history, + quickmarks, bookmarks): + """Test deleting a history entry.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - hist_before = list(web_history) - view = _mock_view_index(model, 2, 0, qtbot) - model.delete_cur_item(view) - assert list(web_history) == hist_before + parent = model.index(2, 0) + idx = model.index(1, 0, parent) + + # sanity checks + assert model.data(parent) == "History" + assert model.data(idx) == 'https://python.org' + + model.delete_cur_item(idx) + assert not web_history_stub.contains('url', 'https://python.org') def test_session_completion(qtmodeltester, session_manager_stub): @@ -431,7 +436,7 @@ def test_tab_completion(qtmodeltester, fake_web_tab, app_stub, win_registry, }) -def test_tab_completion_delete(qtmodeltester, fake_web_tab, qtbot, app_stub, +def test_tab_completion_delete(qtmodeltester, fake_web_tab, app_stub, win_registry, tabbed_browser_stubs): """Verify closing a tab by deleting it from the completion widget.""" tabbed_browser_stubs[0].tabs = [ @@ -447,9 +452,14 @@ def test_tab_completion_delete(qtmodeltester, fake_web_tab, qtbot, app_stub, qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - view = _mock_view_index(model, 0, 1, qtbot) - qtbot.add_widget(view) - model.delete_cur_item(view) + parent = model.index(0, 0) + idx = model.index(1, 0, parent) + + # sanity checks + assert model.data(parent) == "0" + assert model.data(idx) == '0/2' + + model.delete_cur_item(idx) actual = [tab.url() for tab in tabbed_browser_stubs[0].tabs] assert actual == [QUrl('https://github.com'), QUrl('https://duckduckgo.com')] From 62a849c2dbf21b988f6a9391ef1be7ffc7473e25 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 26 Jun 2017 12:41:48 -0400 Subject: [PATCH 134/161] Fix bugs introduced in test_models --- tests/unit/completion/test_models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 5677eab52..77bb89ccf 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -137,6 +137,7 @@ def bookmarks(bookmark_manager_stub): ]) return bookmark_manager_stub + @pytest.fixture def history_completion_table(init_sql): return sql.SqlTable("CompletionHistory", ['url', 'title', 'last_atime']) @@ -593,7 +594,7 @@ def test_url_completion_benchmark(benchmark, config_stub, 'title': ['title{}'.format(i) for i in r] } - web_history_stub.insert_batch(entries) + web_history_stub.completions.insert_batch(entries) quickmark_manager_stub.marks = collections.OrderedDict([ ('title{}'.format(i), 'example.com/{}'.format(i)) From f06880c6e2ac8036c9e39180f669c71a64178f9b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 27 Jun 2017 08:40:43 -0400 Subject: [PATCH 135/161] Fix history completion delete function. In order to update SqlQueryModel's rowCount after re-running the query, we must call setQuery again. --- qutebrowser/completion/models/sqlcategory.py | 3 ++- tests/helpers/stubs.py | 5 ++++- tests/unit/completion/test_models.py | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 9edd5f96e..e20edad3c 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -99,5 +99,6 @@ class SqlCategory(QSqlQueryModel): for i in range(self.columnCount())] self.delete_func(data) # re-run query to reload updated table - with debug.log_time('sql', 'Running completion query'): + with debug.log_time('sql', 'Re-running completion query post-delete'): self._query.run() + self.setQuery(self._query) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index a490ad5bb..6d0160f03 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -532,6 +532,10 @@ class WebHistoryStub(sql.SqlTable): self.completion = sql.SqlTable("CompletionHistory", ['url', 'title', 'last_atime']) + def __contains__(self, url): + q = self.contains_query('url') + return q.run(val=url).value() + def add_url(self, url, title="", *, redirect=False, atime=None): self.insert({'url': url, 'title': title, 'atime': atime, 'redirect': redirect}) @@ -540,7 +544,6 @@ class WebHistoryStub(sql.SqlTable): 'title': title, 'last_atime': atime}) - def delete_url(self, url): """Remove all history entries with the given url. diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 77bb89ccf..1c1a4ce63 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -392,8 +392,9 @@ def test_url_completion_delete_history(qtmodeltester, config_stub, assert model.data(parent) == "History" assert model.data(idx) == 'https://python.org' + assert 'https://python.org' in web_history_stub model.delete_cur_item(idx) - assert not web_history_stub.contains('url', 'https://python.org') + assert 'https://python.org' not in web_history_stub def test_session_completion(qtmodeltester, session_manager_stub): @@ -594,7 +595,7 @@ def test_url_completion_benchmark(benchmark, config_stub, 'title': ['title{}'.format(i) for i in r] } - web_history_stub.completions.insert_batch(entries) + web_history_stub.completion.insert_batch(entries) quickmark_manager_stub.marks = collections.OrderedDict([ ('title{}'.format(i), 'example.com/{}'.format(i)) From 6ac940fa32ded695684e998abcd25a1fbea14b15 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 27 Jun 2017 12:33:51 -0400 Subject: [PATCH 136/161] Fix pylint/coverage errors. Ensure 100% coverage for sqlcategory and history, and fix some linter errors --- qutebrowser/completion/models/sqlcategory.py | 1 - qutebrowser/completion/models/urlmodel.py | 2 +- tests/unit/browser/webkit/test_history.py | 11 +++++++++ .../unit/completion/test_completionwidget.py | 2 +- tests/unit/completion/test_models.py | 3 --- tests/unit/completion/test_sqlcategory.py | 24 +++++++++++++++++++ tests/unit/utils/test_version.py | 1 - 7 files changed, 37 insertions(+), 7 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index e20edad3c..b074980b3 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -21,7 +21,6 @@ import re -from PyQt5.QtCore import QModelIndex from PyQt5.QtSql import QSqlQueryModel from qutebrowser.misc import sql diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 397783d6f..0c63f2582 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -22,7 +22,7 @@ from qutebrowser.completion.models import (completionmodel, listcategory, sqlcategory) from qutebrowser.config import config -from qutebrowser.utils import qtutils, log, objreg +from qutebrowser.utils import log, objreg _URLCOL = 0 diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 21945b537..4b317acab 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -125,6 +125,17 @@ def test_clear_force(qtbot, tmpdir, hist): assert not len(hist) +def test_delete_url(hist): + hist.add_url(QUrl('http://example.com/'), atime=0) + hist.add_url(QUrl('http://example.com/1'), atime=0) + hist.add_url(QUrl('http://example.com/2'), atime=0) + + before = set(hist) + hist.delete_url(QUrl('http://example.com/1')) + diff = before.difference(set(hist)) + assert diff == set([('http://example.com/1', '', 0, False)]) + + @pytest.mark.parametrize('url, atime, title, redirect', [ ('http://www.example.com', 12346, 'the title', False), ('http://www.example.com', 12346, 'the title', True) diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 03f352194..1625dc60a 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -233,4 +233,4 @@ def test_completion_item_del_no_selection(completionview): completionview.set_model(model) with pytest.raises(cmdexc.CommandError, match='No item selected!'): completionview.completion_item_del() - func.assert_not_called + func.assert_not_called() diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 1c1a4ce63..959224398 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -20,16 +20,13 @@ """Tests for completion models.""" import collections -import unittest.mock from datetime import datetime import pytest from PyQt5.QtCore import QUrl -from PyQt5.QtWidgets import QTreeView from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value -from qutebrowser.utils import objreg from qutebrowser.misc import sql diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index ad1ff7e75..951a7d865 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -19,11 +19,14 @@ """Test SQL-based completions.""" +import unittest.mock + import pytest from helpers import utils from qutebrowser.misc import sql from qutebrowser.completion.models import sqlcategory +from qutebrowser.commands import cmdexc pytestmark = pytest.mark.usefixtures('init_sql') @@ -151,3 +154,24 @@ def test_group(): select='a, max(b)', group_by='a') cat.set_pattern('') utils.validate_model(cat, [('bar', 3), ('foo', 2)]) + + +def test_delete_cur_item(): + table = sql.SqlTable('Foo', ['a', 'b']) + table.insert({'a': 'foo', 'b': 1}) + table.insert({'a': 'bar', 'b': 2}) + func = unittest.mock.MagicMock() + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], delete_func=func) + cat.set_pattern('') + cat.delete_cur_item(cat.index(0, 0)) + func.assert_called_with(['foo', 1]) + + +def test_delete_cur_item_no_func(): + table = sql.SqlTable('Foo', ['a', 'b']) + table.insert({'a': 'foo', 'b': 1}) + table.insert({'a': 'bar', 'b': 2}) + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a']) + cat.set_pattern('') + with pytest.raises(cmdexc.CommandError, match='Cannot delete this item'): + cat.delete_cur_item(cat.index(0, 0)) diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 13dfb09d2..c0c731f88 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -811,7 +811,6 @@ def test_chromium_version_unpatched(qapp): def test_version_output(git_commit, frozen, style, with_webkit, known_distribution, stubs, monkeypatch, init_sql): """Test version.version().""" - # pylint: disable=too-many-locals class FakeWebEngineProfile: def httpUserAgent(self): return 'Toaster/4.0.4 Chrome/CHROMIUMVERSION Teapot/4.1.8' From 9c0c1745343f345664efa2e02b12c9da3818039d Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 12:44:02 -0400 Subject: [PATCH 137/161] Use builtin SortFilter regex functionality. With the new completion API, we no longer need a custom filterAcceptsRow function. This was necessary to handle the tree structure of the model, but now we use a separate QSortFilterProxyModel for each category, so the data it filters is flat. We can simplify the code by using the builtin setFilterRegExp. This changes the behavior a little, as now all list categories filter on all columns. This should be beneficial if anything. For example, help topics are now filtered on description in addition to name. This also seems to slightly speed up filtering, according to the url model benchmark. Before: ----------------------------------------------- benchmark: 1 tests ---------------------------------------------- Name (time in s) Min Max Mean StdDev Median IQR Outliers(*) Rounds Iterations ----------------------------------------------------------------------------------------------------------------- test_url_completion_benchmark 1.2806 1.3817 1.3195 0.0390 1.3068 0.0487 1;0 5 1 ----------------------------------------------------------------------------------------------------------------- After: ----------------------------------------------- benchmark: 1 tests ---------------------------------------------- Name (time in s) Min Max Mean StdDev Median IQR Outliers(*) Rounds Iterations ----------------------------------------------------------------------------------------------------------------- test_url_completion_benchmark 1.1183 1.1508 1.1281 0.0132 1.1241 0.0142 1;0 5 1 ----------------------------------------------------------------------------------------------------------------- --- qutebrowser/completion/models/listcategory.py | 40 ++++--------------- qutebrowser/completion/models/miscmodels.py | 4 -- qutebrowser/completion/models/urlmodel.py | 6 +-- 3 files changed, 9 insertions(+), 41 deletions(-) diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index c02783e66..ee9a5e53e 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -21,7 +21,7 @@ import re -from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex +from PyQt5.QtCore import Qt, QSortFilterProxyModel, QModelIndex, QRegExp from PyQt5.QtGui import QStandardItem, QStandardItemModel from qutebrowser.utils import qtutils @@ -32,14 +32,14 @@ class ListCategory(QSortFilterProxyModel): """Expose a list of items as a category for the CompletionModel.""" - def __init__(self, name, items, columns_to_filter=None, - delete_func=None, parent=None): + def __init__(self, name, items, delete_func=None, parent=None): super().__init__(parent) self.name = name self.srcmodel = QStandardItemModel(parent=self) self.pattern = '' - self.pattern_re = None - self.columns_to_filter = columns_to_filter or [0] + # ListCategory filters all columns + self.columns_to_filter = [0, 1, 2] + self.setFilterKeyColumn(-1) for item in items: self.srcmodel.appendRow([QStandardItem(x) for x in item]) self.setSourceModel(self.srcmodel) @@ -55,38 +55,12 @@ class ListCategory(QSortFilterProxyModel): val = re.sub(r' +', r' ', val) # See #1919 val = re.escape(val) val = val.replace(r'\ ', '.*') - self.pattern_re = re.compile(val, re.IGNORECASE) + rx = QRegExp(val, Qt.CaseInsensitive) + self.setFilterRegExp(rx) self.invalidate() sortcol = 0 self.sort(sortcol) - def filterAcceptsRow(self, row, parent): - """Custom filter implementation. - - Override QSortFilterProxyModel::filterAcceptsRow. - - Args: - row: The row of the item. - parent: The parent item QModelIndex. - - Return: - True if self.pattern is contained in item. - """ - if not self.pattern: - return True - - for col in self.columns_to_filter: - idx = self.srcmodel.index(row, col, parent) - if not idx.isValid(): # pragma: no cover - # this is a sanity check not hit by any test case - continue - data = self.srcmodel.data(idx) - if not data: - continue - elif self.pattern_re.search(data): - return True - return False - def lessThan(self, lindex, rindex): """Custom sorting implementation. diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 8338452c9..0af6060e7 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -92,9 +92,6 @@ def buffer(): Used for switching the buffer command. """ - idx_column = 0 - url_column = 1 - text_column = 2 def delete_buffer(data): """Close the selected tab.""" @@ -117,7 +114,6 @@ def buffer(): tab.url().toDisplayString(), tabbed_browser.page_title(idx))) cat = listcategory.ListCategory("{}".format(win_id), tabs, - columns_to_filter=[idx_column, url_column, text_column], delete_func=delete_buffer) model.add_category(cat) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 0c63f2582..4f5fdeae9 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -62,11 +62,9 @@ def url(): bookmarks = objreg.get('bookmark-manager').marks.items() model.add_category(listcategory.ListCategory( - 'Quickmarks', quickmarks, columns_to_filter=[0, 1], - delete_func=_delete_quickmark)) + 'Quickmarks', quickmarks, delete_func=_delete_quickmark)) model.add_category(listcategory.ListCategory( - 'Bookmarks', bookmarks, columns_to_filter=[0, 1], - delete_func=_delete_bookmark)) + 'Bookmarks', bookmarks, delete_func=_delete_bookmark)) # replace 's to avoid breaking the query timefmt = config.get('completion', 'timestamp-format').replace("'", "`") From c007f592b3867d318ccfdd8656b982cb2a317693 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 20:43:42 -0400 Subject: [PATCH 138/161] Use more intuitive argument order in sql.delete. --- qutebrowser/browser/history.py | 4 ++-- qutebrowser/completion/models/sqlcategory.py | 1 - qutebrowser/misc/sql.py | 4 ++-- tests/unit/misc/test_sql.py | 6 +++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e9a6ce980..86c6a6cd9 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -122,8 +122,8 @@ class WebHistory(sql.SqlTable): Args: url: URL string to delete. """ - self.delete(url, 'url') - self.completion.delete(url, 'url') + self.delete('url', url) + self.completion.delete('url', url) @pyqtSlot(QUrl, QUrl, str) def add_from_tab(self, url, requested_url, title): diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index b074980b3..001a844b0 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -78,7 +78,6 @@ class SqlCategory(QSqlQueryModel): Args: pattern: string pattern to filter by. - columns_to_filter: indices of columns to apply pattern to. """ # escape to treat a user input % or _ as a literal, not a wildcard pattern = pattern.replace('%', '\\%') diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 52ae13e11..4db3fb899 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -170,12 +170,12 @@ class SqlTable(QObject): q.run() return q.value() - def delete(self, value, field): + def delete(self, field, value): """Remove all rows for which `field` equals `value`. Args: - value: Key value to delete. field: Field to use as the key. + value: Key value to delete. Return: The number of rows deleted. diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 1f67f0bee..8997afc3b 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -124,12 +124,12 @@ def test_delete(qtbot): table.insert({'name': 'nine', 'val': 9, 'lucky': False}) table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) with pytest.raises(KeyError): - table.delete('nope', 'name') + table.delete('name', 'nope') with qtbot.waitSignal(table.changed): - table.delete('thirteen', 'name') + table.delete('name', 'thirteen') assert list(table) == [('one', 1, False), ('nine', 9, False)] with qtbot.waitSignal(table.changed): - table.delete(False, field='lucky') + table.delete('lucky', False) assert not list(table) From 262b028ee9a13ad9b8afa8f1a9f989da497856e4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 20:49:05 -0400 Subject: [PATCH 139/161] Match error message in lineparser test. --- tests/unit/misc/test_lineparser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/misc/test_lineparser.py b/tests/unit/misc/test_lineparser.py index dcdb724cd..cd4a64274 100644 --- a/tests/unit/misc/test_lineparser.py +++ b/tests/unit/misc/test_lineparser.py @@ -114,7 +114,8 @@ class TestLineParser: def test_double_open(self, lineparser): """Test if save() bails on an already open file.""" with lineparser._open('r'): - with pytest.raises(IOError): + with pytest.raises(IOError, + match="Refusing to double-open LineParser."): lineparser.save() def test_prepare_save(self, tmpdir, lineparser): From fd07c571e5ea77beb7c19ecf593cb26a9913ac5b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 20:54:56 -0400 Subject: [PATCH 140/161] Store pattern in completion view, not model. The pattern property is used for highlighting. It is purely display-related, so it should be in the view rather than the model. --- qutebrowser/completion/completiondelegate.py | 3 ++- qutebrowser/completion/completionwidget.py | 3 +++ qutebrowser/completion/models/completionmodel.py | 4 ---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/qutebrowser/completion/completiondelegate.py b/qutebrowser/completion/completiondelegate.py index 5813d6dcb..b2a933cef 100644 --- a/qutebrowser/completion/completiondelegate.py +++ b/qutebrowser/completion/completiondelegate.py @@ -196,7 +196,8 @@ class CompletionItemDelegate(QStyledItemDelegate): self._doc.setDocumentMargin(2) if index.parent().isValid(): - pattern = index.model().pattern + view = self.parent() + pattern = view.pattern columns_to_filter = index.model().columns_to_filter(index) if index.column() in columns_to_filter and pattern: repl = r'\g<0>' diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 415968131..9b8d31ad6 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -40,6 +40,7 @@ class CompletionView(QTreeView): headers, and children show as flat list. Attributes: + pattern: Current filter pattern, used for highlighting. _win_id: The ID of the window this CompletionView is associated with. _height: The height to use for the CompletionView. _height_perc: Either None or a percentage if height should be relative. @@ -106,6 +107,7 @@ class CompletionView(QTreeView): def __init__(self, win_id, parent=None): super().__init__(parent) + self.pattern = '' self._win_id = win_id # FIXME handle new aliases. # objreg.get('config').changed.connect(self.init_command_completion) @@ -288,6 +290,7 @@ class CompletionView(QTreeView): self.expand(model.index(i, 0)) def set_pattern(self, pattern): + self.pattern = pattern with debug.log_time(log.completion, 'Set pattern {}'.format(pattern)): self.model().set_pattern(pattern) self._resize_columns() diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 55b0546d9..9eb57fe88 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -35,7 +35,6 @@ class CompletionModel(QAbstractItemModel): Attributes: column_widths: The width percentages of the columns used in the completion view. - pattern: Current filter pattern, used for highlighting. _categories: The sub-categories. """ @@ -43,7 +42,6 @@ class CompletionModel(QAbstractItemModel): super().__init__(parent) self.column_widths = column_widths self._categories = [] - self.pattern = '' def _cat_from_idx(self, index): """Return the category pointed to by the given index. @@ -180,8 +178,6 @@ class CompletionModel(QAbstractItemModel): pattern: The filter pattern to set. """ log.completion.debug("Setting completion pattern '{}'".format(pattern)) - # TODO: should pattern be saved in the view layer instead? - self.pattern = pattern for cat in self._categories: cat.set_pattern(pattern) From 1e489325c4129e3c6a51005bddaea3dd7f68bca3 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 20:58:15 -0400 Subject: [PATCH 141/161] Assert if index is invalid in delete_cur_item. CompletionView already checks the index, so an error here shouldn't happen. --- qutebrowser/completion/models/completionmodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 9eb57fe88..a97af2342 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -215,10 +215,10 @@ class CompletionModel(QAbstractItemModel): def delete_cur_item(self, index): """Delete the row at the given index.""" + qtutils.ensure_valid(index) parent = index.parent() cat = self._cat_from_idx(parent) - if not cat: - raise cmdexc.CommandError("No category selected") + assert cat, "CompletionView sent invalid index for deletion" self.beginRemoveRows(parent, index.row(), index.row()) cat.delete_cur_item(cat.index(index.row(), 0)) self.endRemoveRows() From c1f5e77fc69a3c23000416f7c9907dd2bf3e958d Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 21:44:44 -0400 Subject: [PATCH 142/161] Implement "Current" completion for bind. When binding a key, the first row will be the current binding if the key is already bound. This should make it easier for users to tell when they are binding a key that is already bound, and what it is bound to. --- qutebrowser/completion/models/miscmodels.py | 12 +++++++++--- tests/unit/completion/test_models.py | 3 +++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 0af6060e7..00fabb031 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -120,14 +120,20 @@ def buffer(): return model -def bind(_key): +def bind(key): """A CompletionModel filled with all bindable commands and descriptions. Args: - _key: the key being bound. + key: the key being bound. """ - # TODO: offer a 'Current binding' completion based on the key. model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) + cmd_name = objreg.get('key-config').get_bindings_for('normal').get(key) + + if cmd_name: + cmd = cmdutils.cmd_dict.get(cmd_name) + data = [(cmd_name, cmd.desc, key)] + model.add_category(listcategory.ListCategory("Current", data)) + cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) model.add_category(listcategory.ListCategory("Commands", cmdlist)) return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 959224398..b7b688a81 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -567,6 +567,9 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, qtmodeltester.check(model) _check_completions(model, { + "Current": [ + ('stop', 'stop qutebrowser', 's'), + ], "Commands": [ ('drop', 'drop all user data', ''), ('hide', '', ''), From 22880926b1aa1dd1ee97b3a0d49d652223e814f0 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 21:46:09 -0400 Subject: [PATCH 143/161] Fix WebHistoryStub for delete argument change --- tests/helpers/stubs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 6d0160f03..05d91fac0 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -550,8 +550,8 @@ class WebHistoryStub(sql.SqlTable): Args: url: URL string to delete. """ - self.delete(url, 'url') - self.completion.delete(url, 'url') + self.delete('url', url) + self.completion.delete('url', url) class HostBlockerStub: From f2dbff92f49a39a214f89d6f5815b03f921e66a1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 2 Jul 2017 12:53:54 -0400 Subject: [PATCH 144/161] Check for PyQt.QtSql and sqlite in earlyinit. Show a graphical error box with install instructions if PyQt.QtSql is not found, rather than failing with CLI errors. Also show an error box if the sqlite driver is not available. --- qutebrowser/misc/earlyinit.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index 9391288de..95f61708a 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -336,6 +336,13 @@ def check_libraries(backend): "http://pyyaml.org/download/pyyaml/ (py3.4) " "or Install via pip.", pip="PyYAML"), + 'PyQt5.QtSql': + _missing_str("PyQt5.QtSql", + windows="Use the installer by Riverbank computing " + "or the standalone qutebrowser exe. " + "http://www.riverbankcomputing.co.uk/" + "software/pyqt/download5", + pip="PyQt5"), } if backend == 'webengine': modules['PyQt5.QtWebEngineWidgets'] = _missing_str("QtWebEngine", @@ -393,6 +400,12 @@ def check_optimize_flag(): "unexpected behavior may occur.") +def check_sqlite(): + from PyQt5.QtSql import QSqlDatabase + if not QSqlDatabase.isDriverAvailable('SQLITE'): + _die('sqlite driver not available! Is sqlite installed?') + + def set_backend(backend): """Set the objects.backend global to the given backend (as string).""" from qutebrowser.misc import objects @@ -432,4 +445,5 @@ def earlyinit(args): check_libraries(backend) check_ssl_support(backend) check_optimize_flag() + check_sqlite() set_backend(backend) From a34df342081a44c58ab2f1634d02aece97a43c22 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 2 Jul 2017 15:22:55 -0400 Subject: [PATCH 145/161] Fix various test/flake8/pylint errors. --- .../completion/models/completionmodel.py | 1 - qutebrowser/completion/models/miscmodels.py | 1 - tests/unit/completion/test_completionmodel.py | 4 +- tests/unit/completion/test_listcategory.py | 39 +++++++------------ 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index a97af2342..778e0cac9 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -22,7 +22,6 @@ from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from qutebrowser.utils import log, qtutils -from qutebrowser.commands import cmdexc class CompletionModel(QAbstractItemModel): diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 00fabb031..167eccde8 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -92,7 +92,6 @@ def buffer(): Used for switching the buffer command. """ - def delete_buffer(data): """Close the selected tab.""" win_id, tab_index = data[0].split('/') diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 5f4670f97..3439400cb 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -27,7 +27,7 @@ import pytest from PyQt5.QtCore import QModelIndex from qutebrowser.completion.models import completionmodel, listcategory -from qutebrowser.commands import cmdexc +from qutebrowser.utils import qtutils @hypothesis.given(strategies.lists(min_size=0, max_size=3, @@ -91,5 +91,5 @@ def test_delete_cur_item(): def test_delete_cur_item_no_cat(): """Test completion_item_del with no selected category.""" model = completionmodel.CompletionModel() - with pytest.raises(cmdexc.CommandError, match='No category selected'): + with pytest.raises(qtutils.QtValueError): model.delete_cur_item(QModelIndex()) diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index d8d35ea3c..377d9abb7 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -25,35 +25,26 @@ from helpers import utils from qutebrowser.completion.models import listcategory -@pytest.mark.parametrize('pattern, filter_cols, before, after', [ - ('foo', [0], - [('foo', '', ''), ('bar', '', '')], - [('foo', '', '')]), +@pytest.mark.parametrize('pattern, before, after', [ + ('foo', + [('foo', ''), ('bar', '')], + [('foo', '')]), - ('foo', [0], - [('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')], - [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]), + ('foo', + [('foob', ''), ('fooc', ''), ('fooa', '')], + [('fooa', ''), ('foob', ''), ('fooc', '')]), # prefer foobar as it starts with the pattern - ('foo', [0], - [('barfoo', '', ''), ('foobar', '', '')], - [('foobar', '', ''), ('barfoo', '', '')]), + ('foo', + [('barfoo', ''), ('foobar', '')], + [('foobar', ''), ('barfoo', '')]), - ('foo', [1], - [('foo', 'bar', ''), ('bar', 'foo', '')], - [('bar', 'foo', '')]), - - ('foo', [0, 1], - [('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')], - [('foo', 'bar', ''), ('bar', 'foo', '')]), - - ('foo', [0, 1, 2], - [('foo', '', ''), ('bar', '')], - [('foo', '', '')]), + ('foo', + [('foo', 'bar'), ('bar', 'foo'), ('bar', 'bar')], + [('foo', 'bar'), ('bar', 'foo')]), ]) -def test_set_pattern(pattern, filter_cols, before, after): +def test_set_pattern(pattern, before, after): """Validate the filtering and sorting results of set_pattern.""" - cat = listcategory.ListCategory('Foo', before, - columns_to_filter=filter_cols) + cat = listcategory.ListCategory('Foo', before) cat.set_pattern(pattern) utils.validate_model(cat, after) From 25c79bec675320f0a1accb2042d5c06e566a9f51 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 3 Jul 2017 08:15:06 -0400 Subject: [PATCH 146/161] Check correct SQL driver in earlyinit. --- qutebrowser/misc/earlyinit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index 95f61708a..eaa493cad 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -402,7 +402,7 @@ def check_optimize_flag(): def check_sqlite(): from PyQt5.QtSql import QSqlDatabase - if not QSqlDatabase.isDriverAvailable('SQLITE'): + if not QSqlDatabase.isDriverAvailable('QSQLITE'): _die('sqlite driver not available! Is sqlite installed?') From 1e1335aa5edf72076a9e25fd129d6e074cba4af6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 3 Jul 2017 08:20:31 -0400 Subject: [PATCH 147/161] Make various SQL code review changes. - Fix outdated comments - Use mock specs when possible - More precise error message check in test_import_txt_invalid. - Fix copyright message - Tweak missing pyqt error message - Dead code: remove group_by and where from sqlcategory. With the new separate completion table, these are no longer used. - Move test_history out of webkit/. History is no longer purely webkit related, it could be webengine. --- qutebrowser/browser/qutescheme.py | 1 - qutebrowser/completion/models/listcategory.py | 10 ++++---- qutebrowser/completion/models/sqlcategory.py | 7 +----- qutebrowser/misc/earlyinit.py | 7 +----- qutebrowser/misc/sql.py | 2 +- .../unit/browser/{webkit => }/test_history.py | 11 ++++++--- tests/unit/completion/test_completionmodel.py | 11 +++++---- .../unit/completion/test_completionwidget.py | 6 ++--- tests/unit/completion/test_listcategory.py | 2 +- tests/unit/completion/test_sqlcategory.py | 23 +------------------ tests/unit/misc/test_lineparser.py | 3 ++- 11 files changed, 29 insertions(+), 54 deletions(-) rename tests/unit/browser/{webkit => }/test_history.py (97%) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 0c495d28b..e25b7df07 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -194,7 +194,6 @@ def history_data(start_time, offset=None): """ hist = objreg.get('web-history') if offset is not None: - # end is 24hrs earlier than start entries = hist.entries_before(start_time, limit=1000, offset=offset) else: # end is 24hrs earlier than start diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index ee9a5e53e..187ebcad6 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -36,7 +36,7 @@ class ListCategory(QSortFilterProxyModel): super().__init__(parent) self.name = name self.srcmodel = QStandardItemModel(parent=self) - self.pattern = '' + self._pattern = '' # ListCategory filters all columns self.columns_to_filter = [0, 1, 2] self.setFilterKeyColumn(-1) @@ -51,7 +51,7 @@ class ListCategory(QSortFilterProxyModel): Args: val: The value to set. """ - self.pattern = val + self._pattern = val val = re.sub(r' +', r' ', val) # See #1919 val = re.escape(val) val = val.replace(r'\ ', '.*') @@ -64,7 +64,7 @@ class ListCategory(QSortFilterProxyModel): def lessThan(self, lindex, rindex): """Custom sorting implementation. - Prefers all items which start with self.pattern. Other than that, uses + Prefers all items which start with self._pattern. Other than that, uses normal Python string sorting. Args: @@ -80,8 +80,8 @@ class ListCategory(QSortFilterProxyModel): left = self.srcmodel.data(lindex) right = self.srcmodel.data(rindex) - leftstart = left.startswith(self.pattern) - rightstart = right.startswith(self.pattern) + leftstart = left.startswith(self._pattern) + rightstart = right.startswith(self._pattern) if leftstart and rightstart: return left < right diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 001a844b0..4819e940b 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -33,7 +33,7 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, title=None, filter_fields, sort_by=None, - sort_order=None, select='*', where=None, group_by=None, + sort_order=None, select='*', delete_func=None, parent=None): """Create a new completion category backed by a sql table. @@ -42,7 +42,6 @@ class SqlCategory(QSqlQueryModel): title: Title of category, defaults to table name. filter_fields: Names of fields to apply filter pattern to. select: A custom result column expression for the select statement. - where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Either 'asc' or 'desc', if sort_by is non-None delete_func: Callback to delete a selected item. @@ -57,10 +56,6 @@ class SqlCategory(QSqlQueryModel): for f in filter_fields) querystr += ')' - if where: - querystr += ' and ' + where - if group_by: - querystr += ' group by {}'.format(group_by) if sort_by: assert sort_order in ['asc', 'desc'], sort_order querystr += ' order by {} {}'.format(sort_by, sort_order) diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index eaa493cad..21be8ed37 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -337,12 +337,7 @@ def check_libraries(backend): "or Install via pip.", pip="PyYAML"), 'PyQt5.QtSql': - _missing_str("PyQt5.QtSql", - windows="Use the installer by Riverbank computing " - "or the standalone qutebrowser exe. " - "http://www.riverbankcomputing.co.uk/" - "software/pyqt/download5", - pip="PyQt5"), + _missing_str("PyQt5.QtSql") } if backend == 'webengine': modules['PyQt5.QtWebEngineWidgets'] = _missing_str("QtWebEngine", diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 4db3fb899..35c5359dd 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -209,8 +209,8 @@ class SqlTable(QObject): """Performantly append multiple rows to the table. Args: - rows: A list of lists, where each sub-list is a row. values: A dict with a list of values to insert for each field name. + replace: If true, overwrite rows with a primary key match. """ q = self._insert_query(values, replace) for key, val in values.items(): diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/test_history.py similarity index 97% rename from tests/unit/browser/webkit/test_history.py rename to tests/unit/browser/test_history.py index 4b317acab..9496accd2 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/test_history.py @@ -113,7 +113,8 @@ def test_clear(qtbot, tmpdir, hist, mocker): hist.add_url(QUrl('http://example.com/')) hist.add_url(QUrl('http://www.qutebrowser.org/')) - m = mocker.patch('qutebrowser.browser.history.message.confirm_async') + m = mocker.patch('qutebrowser.browser.history.message.confirm_async', + spec=[]) hist.clear() assert m.called @@ -288,15 +289,19 @@ def test_import_txt_skip(hist, data_tmpdir, line, monkeypatch, stubs): '68891-x http://example.com/bad-flag', '68891 http://.com', ]) -def test_import_txt_invalid(hist, data_tmpdir, line, monkeypatch, stubs): +def test_import_txt_invalid(hist, data_tmpdir, line, monkeypatch, stubs, + caplog): """import_txt should fail on certain lines.""" monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) histfile = data_tmpdir / 'history' histfile.write(line) - with pytest.raises(Exception): + with caplog.at_level(logging.ERROR): hist.import_txt() + assert any(rec.msg.startswith("Failed to import history:") + for rec in caplog.records) + assert histfile.exists() diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 3439400cb..bc7f15ab8 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -36,7 +36,7 @@ def test_first_last_item(counts): """Test that first() and last() index to the first and last items.""" model = completionmodel.CompletionModel() for c in counts: - cat = mock.Mock() + cat = mock.Mock(spec=['layoutChanged']) cat.rowCount = mock.Mock(return_value=c) model.add_category(cat) nonempty = [i for i, rowCount in enumerate(counts) if rowCount > 0] @@ -70,16 +70,17 @@ def test_count(counts): def test_set_pattern(pat): """Validate the filtering and sorting results of set_pattern.""" model = completionmodel.CompletionModel() - cats = [mock.Mock(spec=['set_pattern', 'layoutChanged'])] * 3 + cats = [mock.Mock(spec=['set_pattern', 'layoutChanged']) for _ in range(3)] for c in cats: - c.set_pattern = mock.Mock() + c.set_pattern = mock.Mock(spec=[]) model.add_category(c) model.set_pattern(pat) - assert all(c.set_pattern.called_with([pat]) for c in cats) + for c in cats: + c.set_pattern.assert_called_with(pat) def test_delete_cur_item(): - func = mock.Mock() + func = mock.Mock(spec=[]) model = completionmodel.CompletionModel() cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=func) model.add_category(cat) diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 1625dc60a..cad45b5c1 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -83,7 +83,7 @@ def test_set_model(completionview): def test_set_pattern(completionview): model = completionmodel.CompletionModel() - model.set_pattern = mock.Mock() + model.set_pattern = mock.Mock(spec=[]) completionview.set_model(model) completionview.set_pattern('foo') model.set_pattern.assert_called_with('foo') @@ -214,7 +214,7 @@ def test_completion_show(show, rows, quick_complete, completionview, def test_completion_item_del(completionview): """Test that completion_item_del invokes delete_cur_item in the model.""" - func = mock.Mock() + func = mock.Mock(spec=[]) model = completionmodel.CompletionModel() cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=func) model.add_category(cat) @@ -226,7 +226,7 @@ def test_completion_item_del(completionview): def test_completion_item_del_no_selection(completionview): """Test that completion_item_del with an invalid index.""" - func = mock.Mock() + func = mock.Mock(spec=[]) model = completionmodel.CompletionModel() cat = listcategory.ListCategory('', [('foo',)], delete_func=func) model.add_category(cat) diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index 377d9abb7..5b6a5eea4 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2015-2016 Florian Bruhin (The Compiler) +# Copyright 2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 951a7d865..10d96f571 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -135,32 +135,11 @@ def test_select(): utils.validate_model(cat, [('bar', 'baz', 'foo')]) -def test_where(): - table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert({'a': 'foo', 'b': 'bar', 'c': False}) - table.insert({'a': 'baz', 'b': 'biz', 'c': True}) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') - cat.set_pattern('') - utils.validate_model(cat, [('foo', 'bar', False)]) - - -def test_group(): - table = sql.SqlTable('Foo', ['a', 'b']) - table.insert({'a': 'foo', 'b': 1}) - table.insert({'a': 'bar', 'b': 3}) - table.insert({'a': 'foo', 'b': 2}) - table.insert({'a': 'bar', 'b': 0}) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], - select='a, max(b)', group_by='a') - cat.set_pattern('') - utils.validate_model(cat, [('bar', 3), ('foo', 2)]) - - def test_delete_cur_item(): table = sql.SqlTable('Foo', ['a', 'b']) table.insert({'a': 'foo', 'b': 1}) table.insert({'a': 'bar', 'b': 2}) - func = unittest.mock.MagicMock() + func = unittest.mock.Mock(spec=[]) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], delete_func=func) cat.set_pattern('') cat.delete_cur_item(cat.index(0, 0)) diff --git a/tests/unit/misc/test_lineparser.py b/tests/unit/misc/test_lineparser.py index cd4a64274..0c78035b7 100644 --- a/tests/unit/misc/test_lineparser.py +++ b/tests/unit/misc/test_lineparser.py @@ -58,7 +58,8 @@ class TestBaseLineParser: mocker.patch('builtins.open', mock.mock_open()) with lineparser._open('r'): - with pytest.raises(IOError): + with pytest.raises(IOError, + match="Refusing to double-open LineParser."): with lineparser._open('r'): pass From 81f5b7115f6600ad8ba178c4b2b1a74f97120b94 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 5 Jul 2017 08:44:56 -0400 Subject: [PATCH 148/161] Add spec=[] to two mock functions in tests. --- tests/unit/completion/test_completionmodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index bc7f15ab8..2c0af3a4b 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -37,7 +37,7 @@ def test_first_last_item(counts): model = completionmodel.CompletionModel() for c in counts: cat = mock.Mock(spec=['layoutChanged']) - cat.rowCount = mock.Mock(return_value=c) + cat.rowCount = mock.Mock(return_value=c, spec=[]) model.add_category(cat) nonempty = [i for i, rowCount in enumerate(counts) if rowCount > 0] if not nonempty: @@ -61,7 +61,7 @@ def test_count(counts): model = completionmodel.CompletionModel() for c in counts: cat = mock.Mock(spec=['rowCount', 'layoutChanged']) - cat.rowCount = mock.Mock(return_value=c) + cat.rowCount = mock.Mock(return_value=c, spec=[]) model.add_category(cat) assert model.count() == sum(counts) From cee0aa3adc01faed0f18e982fb3a37e04c8a469f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 6 Jul 2017 07:36:59 -0400 Subject: [PATCH 149/161] Show error dialog is sql isn't available. If creating the sql database fails, show an error dialog assuming sqlite is not installed. This removes the isDriverAvailable check as it was true even with sqlite uninstalled. sql.version now inits itself if sql is not already initialized and prints 'UNAVAILABLE ()' if init fails. This is to avoid cascading errors, where one error would create a crash dialog, which calls sql.version, which would create another error. --- qutebrowser/app.py | 8 ++++++-- qutebrowser/misc/earlyinit.py | 7 ------- qutebrowser/misc/sql.py | 14 +++++++++++--- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 677f79a7b..46c7578bd 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -85,7 +85,6 @@ def run(args): if args.version: # we need to init sql to print the sql version # we can use an in-memory database as we just want to query the version - sql.init(':memory:') print(version.version()) sys.exit(usertypes.Exit.ok) @@ -429,7 +428,12 @@ def _init_modules(args, crash_handler): keyconf.init(qApp) log.init.debug("Initializing sql...") - sql.init(os.path.join(standarddir.data(), 'history.sqlite')) + try: + sql.init(os.path.join(standarddir.data(), 'history.sqlite')) + except sql.SqlException as e: + error.handle_fatal_exc(e, args, 'Is sqlite installed?', + pre_text='Failed to initialize SQL') + sys.exit(usertypes.Exit.err_init) log.init.debug("Initializing web history...") history.init(qApp) diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index 68f5db71e..fb09b4ab2 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -396,12 +396,6 @@ def check_optimize_flag(): "unexpected behavior may occur.") -def check_sqlite(): - from PyQt5.QtSql import QSqlDatabase - if not QSqlDatabase.isDriverAvailable('QSQLITE'): - _die('sqlite driver not available! Is sqlite installed?') - - def set_backend(backend): """Set the objects.backend global to the given backend (as string).""" from qutebrowser.misc import objects @@ -441,5 +435,4 @@ def earlyinit(args): check_libraries(backend) check_ssl_support(backend) check_optimize_flag() - check_sqlite() set_backend(backend) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 35c5359dd..4ae37f01f 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -37,6 +37,8 @@ class SqlException(Exception): def init(db_path): """Initialize the SQL database connection.""" database = QSqlDatabase.addDatabase('QSQLITE') + if not database.isValid(): + raise SqlException('Failed to add database. Is sqlite installed?') database.setDatabaseName(db_path) if not database.open(): raise SqlException("Failed to open sqlite database at {}: {}" @@ -50,9 +52,15 @@ def close(): def version(): """Return the sqlite version string.""" - q = Query("select sqlite_version()") - q.run() - return q.value() + try: + if not QSqlDatabase.database().isOpen(): + init(':memory:') + ver = Query("select sqlite_version()").run().value() + close() + return ver + return Query("select sqlite_version()").run().value() + except SqlException as e: + return 'UNAVAILABLE ({})'.format(e) class Query(QSqlQuery): From 1dd5f06a4fa6e7b57af10ed3deeda7c4db119896 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 6 Jul 2017 08:02:16 -0400 Subject: [PATCH 150/161] Fix debug-dump-history behavior. Ensure the file is closed before printing the success message. This will hopefully fix the AppVeyor tests. --- qutebrowser/browser/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 86c6a6cd9..e583fba40 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -282,7 +282,7 @@ class WebHistory(sql.SqlTable): try: with open(dest, 'w', encoding='utf-8') as f: f.write('\n'.join(lines)) - message.info("Dumped history to {}".format(dest)) + message.info("Dumped history to {}".format(dest)) except OSError as e: raise cmdexc.CommandError('Could not write history: {}', e) From f9f8900fe90c48e7a36d2d4108c25056275d23c7 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 7 Jul 2017 21:16:50 -0400 Subject: [PATCH 151/161] More sql code review fixes. - remove outdated comment - fix sql init error message - clean up history text import code - fix test_history file path in coverage check - use real web history, not stub, for completion model tests - use qtmodeltester in sql/list_category tests - test url encoding in history tests - fix test_clear by using a callable mock - remove test_debug_dump_history_oserror as the check is now the same as for the file not existing - rename nonempty to data in test_completionmodel - add more delete_cur_item tests - test empty option/value completion --- qutebrowser/app.py | 6 +- qutebrowser/browser/history.py | 11 +--- scripts/dev/check_coverage.py | 6 +- tests/helpers/fixtures.py | 28 ++++++--- tests/helpers/stubs.py | 32 ---------- tests/helpers/utils.py | 13 ---- tests/unit/browser/test_history.py | 31 +++++----- tests/unit/completion/test_completionmodel.py | 12 ++-- tests/unit/completion/test_listcategory.py | 26 +++++++- tests/unit/completion/test_models.py | 60 ++++++++++++------- tests/unit/completion/test_sqlcategory.py | 13 ++-- 11 files changed, 115 insertions(+), 123 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 46c7578bd..da66efe24 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -83,8 +83,6 @@ def run(args): standarddir.init(args) if args.version: - # we need to init sql to print the sql version - # we can use an in-memory database as we just want to query the version print(version.version()) sys.exit(usertypes.Exit.ok) @@ -431,8 +429,8 @@ def _init_modules(args, crash_handler): try: sql.init(os.path.join(standarddir.data(), 'history.sqlite')) except sql.SqlException as e: - error.handle_fatal_exc(e, args, 'Is sqlite installed?', - pre_text='Failed to initialize SQL') + error.handle_fatal_exc(e, args, 'Error initializing SQL', + pre_text='Error initializing SQL') sys.exit(usertypes.Exit.err_init) log.init.debug("Initializing web history...") diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e583fba40..b5b8af149 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -171,6 +171,8 @@ class WebHistory(sql.SqlTable): def _parse_entry(self, line): """Parse a history line like '12345 http://example.com title'.""" + if not line or line.startswith('#'): + return None data = line.split(maxsplit=2) if len(data) == 2: atime, url = data @@ -240,11 +242,8 @@ class WebHistory(sql.SqlTable): data = {'url': [], 'title': [], 'atime': [], 'redirect': []} completion_data = {'url': [], 'title': [], 'last_atime': []} for (i, line) in enumerate(f): - line = line.strip() - if not line or line.startswith('#'): - continue try: - parsed = self._parse_entry(line) + parsed = self._parse_entry(line.strip()) if parsed is None: continue url, title, atime, redirect = parsed @@ -271,10 +270,6 @@ class WebHistory(sql.SqlTable): """ dest = os.path.expanduser(dest) - dirname = os.path.dirname(dest) - if not os.path.exists(dirname): - raise cmdexc.CommandError('Path does not exist', dirname) - lines = ('{}{} {} {}' .format(int(x.atime), '-r' * x.redirect, x.url, x.title) for x in self.select(sort_by='atime', sort_order='asc')) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index c01ae64af..950f21ff1 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -51,12 +51,10 @@ PERFECT_FILES = [ 'browser/webkit/cache.py'), ('tests/unit/browser/webkit/test_cookies.py', 'browser/webkit/cookies.py'), - ('tests/unit/browser/webkit/test_history.py', + ('tests/unit/browser/test_history.py', 'browser/history.py'), - ('tests/unit/browser/webkit/test_history.py', + ('tests/unit/browser/test_history.py', 'browser/webkit/webkithistory.py'), - ('tests/unit/browser/webkit/test_history.py', - 'browser/history.py'), ('tests/unit/browser/webkit/http/test_http.py', 'browser/webkit/http.py'), ('tests/unit/browser/webkit/http/test_content_disposition.py', diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 4fd6a0731..dd689a531 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -257,15 +257,6 @@ def bookmark_manager_stub(stubs): objreg.delete('bookmark-manager') -@pytest.fixture -def web_history_stub(init_sql, stubs): - """Fixture which provides a fake web-history object.""" - stub = stubs.WebHistoryStub() - objreg.register('web-history', stub) - yield stub - objreg.delete('web-history') - - @pytest.fixture def session_manager_stub(stubs): """Fixture which provides a fake session-manager object.""" @@ -491,3 +482,22 @@ def init_sql(data_tmpdir): sql.init(path) yield sql.close() + + +@pytest.fixture +def validate_model(qtmodeltester): + """Provides a function to validate a completion category.""" + def validate(cat, expected): + """Check that a category contains the items in the given order. + + Args: + cat: The category to inspect. + expected: A list of tuples containing the expected items. + """ + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(cat) + assert cat.rowCount() == len(expected) + for row, items in enumerate(expected): + for col, item in enumerate(items): + assert cat.data(cat.index(row, col)) == item + return validate diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 61dcefd66..47d254f86 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -33,7 +33,6 @@ from qutebrowser.browser import browsertab from qutebrowser.config import configexc from qutebrowser.utils import usertypes, utils from qutebrowser.mainwindow import mainwindow -from qutebrowser.misc import sql class FakeNetworkCache(QAbstractNetworkCache): @@ -523,37 +522,6 @@ class QuickmarkManagerStub(UrlMarkManagerStub): self.delete(key) -class WebHistoryStub(sql.SqlTable): - - """Stub for the web-history object.""" - - def __init__(self): - super().__init__("History", ['url', 'title', 'atime', 'redirect']) - self.completion = sql.SqlTable("CompletionHistory", - ['url', 'title', 'last_atime']) - - def __contains__(self, url): - q = self.contains_query('url') - return q.run(val=url).value() - - def add_url(self, url, title="", *, redirect=False, atime=None): - self.insert({'url': url, 'title': title, 'atime': atime, - 'redirect': redirect}) - if not redirect: - self.completion.insert({'url': url, - 'title': title, - 'last_atime': atime}) - - def delete_url(self, url): - """Remove all history entries with the given url. - - Args: - url: URL string to delete. - """ - self.delete('url', url) - self.completion.delete('url', url) - - class HostBlockerStub: """Stub for the host-blocker object.""" diff --git a/tests/helpers/utils.py b/tests/helpers/utils.py index 051b7bd50..7dbd7dd25 100644 --- a/tests/helpers/utils.py +++ b/tests/helpers/utils.py @@ -170,16 +170,3 @@ def abs_datapath(): """Get the absolute path to the end2end data directory.""" file_abs = os.path.abspath(os.path.dirname(__file__)) return os.path.join(file_abs, '..', 'end2end', 'data') - - -def validate_model(cat, expected): - """Check that a category contains the expected items in the given order. - - Args: - cat: The category to inspect. - expected: A list of tuples containing the expected items. - """ - assert cat.rowCount() == len(expected) - for row, items in enumerate(expected): - for col, item in enumerate(items): - assert cat.data(cat.index(row, col)) == item diff --git a/tests/unit/browser/test_history.py b/tests/unit/browser/test_history.py index 9496accd2..911184dc0 100644 --- a/tests/unit/browser/test_history.py +++ b/tests/unit/browser/test_history.py @@ -21,6 +21,7 @@ import logging import os +import unittest.mock import pytest from PyQt5.QtCore import QUrl @@ -114,7 +115,7 @@ def test_clear(qtbot, tmpdir, hist, mocker): hist.add_url(QUrl('http://www.qutebrowser.org/')) m = mocker.patch('qutebrowser.browser.history.message.confirm_async', - spec=[]) + new=unittest.mock.Mock, spec=[]) hist.clear() assert m.called @@ -134,16 +135,22 @@ def test_delete_url(hist): before = set(hist) hist.delete_url(QUrl('http://example.com/1')) diff = before.difference(set(hist)) - assert diff == set([('http://example.com/1', '', 0, False)]) + assert diff == {('http://example.com/1', '', 0, False)} -@pytest.mark.parametrize('url, atime, title, redirect', [ - ('http://www.example.com', 12346, 'the title', False), - ('http://www.example.com', 12346, 'the title', True) +@pytest.mark.parametrize('url, atime, title, redirect, expected_url', [ + ('http://www.example.com', 12346, 'the title', False, + 'http://www.example.com'), + ('http://www.example.com', 12346, 'the title', True, + 'http://www.example.com'), + ('http://www.example.com/spa ce', 12346, 'the title', False, + 'http://www.example.com/spa%20ce'), + ('https://user:pass@example.com', 12346, 'the title', False, + 'https://user@example.com'), ]) -def test_add_item(qtbot, hist, url, atime, title, redirect): +def test_add_item(qtbot, hist, url, atime, title, redirect, expected_url): hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) - assert list(hist) == [(url, title, atime, redirect)] + assert list(hist) == [(expected_url, title, atime, redirect)] def test_add_item_invalid(qtbot, hist, caplog): @@ -164,7 +171,7 @@ def test_add_item_invalid(qtbot, hist, caplog): def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog): with caplog.at_level(level): hist.add_from_tab(QUrl(url), QUrl(req_url), 'title') - assert set(list(hist)) == set(expected) + assert set(hist) == set(expected) @pytest.fixture @@ -330,11 +337,3 @@ def test_debug_dump_history_nonexistent(hist, tmpdir): histfile = tmpdir / 'nonexistent' / 'history' with pytest.raises(cmdexc.CommandError): hist.debug_dump_history(str(histfile)) - - -def test_debug_dump_history_oserror(hist, tmpdir): - histfile = tmpdir / 'history' - histfile.write('') - os.chmod(str(histfile), 0) - with pytest.raises(cmdexc.CommandError): - hist.debug_dump_history(str(histfile)) diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 2c0af3a4b..ce0fe3765 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -39,18 +39,18 @@ def test_first_last_item(counts): cat = mock.Mock(spec=['layoutChanged']) cat.rowCount = mock.Mock(return_value=c, spec=[]) model.add_category(cat) - nonempty = [i for i, rowCount in enumerate(counts) if rowCount > 0] - if not nonempty: + data = [i for i, rowCount in enumerate(counts) if rowCount > 0] + if not data: # with no items, first and last should be an invalid index assert not model.first_item().isValid() assert not model.last_item().isValid() else: - first = nonempty[0] - last = nonempty[-1] - # first item of the first nonempty category + first = data[0] + last = data[-1] + # first item of the first data category assert model.first_item().row() == 0 assert model.first_item().parent().row() == first - # last item of the last nonempty category + # last item of the last data category assert model.last_item().row() == counts[last] - 1 assert model.last_item().parent().row() == last diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index 5b6a5eea4..06c7ac1a6 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -19,10 +19,12 @@ """Tests for CompletionFilterModel.""" +from unittest import mock + import pytest -from helpers import utils from qutebrowser.completion.models import listcategory +from qutebrowser.commands import cmdexc @pytest.mark.parametrize('pattern, before, after', [ @@ -43,8 +45,26 @@ from qutebrowser.completion.models import listcategory [('foo', 'bar'), ('bar', 'foo'), ('bar', 'bar')], [('foo', 'bar'), ('bar', 'foo')]), ]) -def test_set_pattern(pattern, before, after): +def test_set_pattern(pattern, before, after, validate_model): """Validate the filtering and sorting results of set_pattern.""" cat = listcategory.ListCategory('Foo', before) cat.set_pattern(pattern) - utils.validate_model(cat, after) + validate_model(cat, after) + + +def test_delete_cur_item(validate_model): + func = mock.Mock(spec=[]) + cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')], + delete_func=func) + idx = cat.index(0, 0) + cat.delete_cur_item(idx) + func.assert_called_once_with(['a', 'b']) + validate_model(cat, [('c', 'd')]) + + +def test_delete_cur_item_no_func(validate_model): + cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')]) + idx = cat.index(0, 0) + with pytest.raises(cmdexc.CommandError, match="Cannot delete this item."): + cat.delete_cur_item(idx) + validate_model(cat, [('a', 'b'), ('c', 'd')]) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index b7b688a81..4fab0c402 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -27,7 +27,8 @@ from PyQt5.QtCore import QUrl from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value -from qutebrowser.misc import sql +from qutebrowser.utils import objreg +from qutebrowser.browser import history def _check_completions(model, expected): @@ -136,29 +137,33 @@ def bookmarks(bookmark_manager_stub): @pytest.fixture -def history_completion_table(init_sql): - return sql.SqlTable("CompletionHistory", ['url', 'title', 'last_atime']) +def web_history(init_sql, stubs): + """Fixture which provides a web-history object.""" + stub = history.WebHistory() + objreg.register('web-history', stub) + yield stub + objreg.delete('web-history') @pytest.fixture -def web_history(web_history_stub): +def web_history_populated(web_history): """Pre-populate the web-history database.""" - web_history_stub.add_url( - url='http://qutebrowser.org', + web_history.add_url( + url=QUrl('http://qutebrowser.org'), title='qutebrowser', atime=datetime(2015, 9, 5).timestamp() ) - web_history_stub.add_url( - url='https://python.org', + web_history.add_url( + url=QUrl('https://python.org'), title='Welcome to Python.org', atime=datetime(2016, 3, 8).timestamp() ) - web_history_stub.add_url( - url='https://github.com', + web_history.add_url( + url=QUrl('https://github.com'), title='https://github.com', atime=datetime(2016, 5, 1).timestamp() ) - return web_history_stub + return web_history def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, @@ -261,8 +266,8 @@ def test_bookmark_completion(qtmodeltester, bookmarks): }) -def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, - bookmarks): +def test_url_completion(qtmodeltester, config_stub, web_history_populated, + quickmarks, bookmarks): """Test the results of url completion. Verify that: @@ -313,12 +318,12 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, ('foo%bar', '', '%', 1), ('foobar', '', '%', 0), ]) -def test_url_completion_pattern(config_stub, web_history_stub, +def test_url_completion_pattern(config_stub, web_history, quickmark_manager_stub, bookmark_manager_stub, url, title, pattern, rowcount): """Test that url completion filters by url and title.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} - web_history_stub.add_url(url, title) + web_history.add_url(QUrl(url), title) model = urlmodel.url() model.set_pattern(pattern) # 2, 0 is History @@ -349,7 +354,7 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, bookmarks, def test_url_completion_delete_quickmark(qtmodeltester, config_stub, - web_history, quickmarks, bookmarks, + quickmarks, web_history, bookmarks, qtbot): """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} @@ -373,7 +378,7 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, def test_url_completion_delete_history(qtmodeltester, config_stub, - web_history_stub, web_history, + web_history_populated, quickmarks, bookmarks): """Test deleting a history entry.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} @@ -389,9 +394,9 @@ def test_url_completion_delete_history(qtmodeltester, config_stub, assert model.data(parent) == "History" assert model.data(idx) == 'https://python.org' - assert 'https://python.org' in web_history_stub + assert 'https://python.org' in web_history_populated model.delete_cur_item(idx) - assert 'https://python.org' not in web_history_stub + assert 'https://python.org' not in web_history_populated def test_session_completion(qtmodeltester, session_manager_stub): @@ -504,6 +509,12 @@ def test_setting_option_completion(qtmodeltester, monkeypatch, stubs, }) +def test_setting_option_completion_empty(monkeypatch, stubs, config_stub): + module = 'qutebrowser.completion.models.configmodel' + _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') + assert configmodel.option('typo') is None + + def test_setting_option_completion_valuelist(qtmodeltester, monkeypatch, stubs, config_stub): module = 'qutebrowser.completion.models.configmodel' @@ -545,6 +556,13 @@ def test_setting_value_completion(qtmodeltester, monkeypatch, stubs, }) +def test_setting_value_completion_empty(monkeypatch, stubs, config_stub): + module = 'qutebrowser.completion.models.configmodel' + _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') + config_stub.data = {'general': {}} + assert configmodel.value('general', 'typo') is None + + def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, key_config_stub): """Test the results of keybinding command completion. @@ -583,7 +601,7 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, def test_url_completion_benchmark(benchmark, config_stub, quickmark_manager_stub, bookmark_manager_stub, - web_history_stub): + web_history): """Benchmark url completion.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 1000} @@ -595,7 +613,7 @@ def test_url_completion_benchmark(benchmark, config_stub, 'title': ['title{}'.format(i) for i in r] } - web_history_stub.completion.insert_batch(entries) + web_history.completion.insert_batch(entries) quickmark_manager_stub.marks = collections.OrderedDict([ ('title{}'.format(i), 'example.com/{}'.format(i)) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 10d96f571..c20f8f1c5 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -23,7 +23,6 @@ import unittest.mock import pytest -from helpers import utils from qutebrowser.misc import sql from qutebrowser.completion.models import sqlcategory from qutebrowser.commands import cmdexc @@ -61,14 +60,14 @@ pytestmark = pytest.mark.usefixtures('init_sql') [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), ]) -def test_sorting(sort_by, sort_order, data, expected): +def test_sorting(sort_by, sort_order, data, expected, validate_model): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) cat.set_pattern('') - utils.validate_model(cat, expected) + validate_model(cat, expected) @pytest.mark.parametrize('pattern, filter_cols, before, after', [ @@ -116,7 +115,7 @@ def test_sorting(sort_by, sort_order, data, expected): [("can't touch this", '', ''), ('a', '', '')], [("can't touch this", '', '')]), ]) -def test_set_pattern(pattern, filter_cols, before, after): +def test_set_pattern(pattern, filter_cols, before, after, validate_model): """Validate the filtering and sorting results of set_pattern.""" table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: @@ -124,15 +123,15 @@ def test_set_pattern(pattern, filter_cols, before, after): filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) cat.set_pattern(pattern) - utils.validate_model(cat, after) + validate_model(cat, after) -def test_select(): +def test_select(validate_model): table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') cat.set_pattern('') - utils.validate_model(cat, [('bar', 'baz', 'foo')]) + validate_model(cat, [('bar', 'baz', 'foo')]) def test_delete_cur_item(): From cf4ac1a5b71c23a7b1e3aecb7fc110f5cde81f86 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 8 Jul 2017 16:34:38 -0400 Subject: [PATCH 152/161] SQL code review changes. - use mocker.Mock instead of mock.Mock to avoid an extra import - attach model to validator sooner so it can validate changes in the model during the test --- tests/helpers/fixtures.py | 33 +++++++++++++--------- tests/unit/browser/test_history.py | 3 +- tests/unit/completion/test_listcategory.py | 15 ++++++---- tests/unit/completion/test_sqlcategory.py | 15 ++++++---- 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index dd689a531..e38a3ce49 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -484,20 +484,25 @@ def init_sql(data_tmpdir): sql.close() -@pytest.fixture -def validate_model(qtmodeltester): - """Provides a function to validate a completion category.""" - def validate(cat, expected): - """Check that a category contains the items in the given order. +class ModelValidator: - Args: - cat: The category to inspect. - expected: A list of tuples containing the expected items. - """ - qtmodeltester.data_display_may_return_none = True - qtmodeltester.check(cat) - assert cat.rowCount() == len(expected) + """Validates completion models.""" + + def __init__(self, modeltester): + modeltester.data_display_may_return_none = True + self._modeltester = modeltester + + def set_model(self, model): + self._model = model + self._modeltester.check(model) + + def validate(self, expected): + assert self._model.rowCount() == len(expected) for row, items in enumerate(expected): for col, item in enumerate(items): - assert cat.data(cat.index(row, col)) == item - return validate + assert self._model.data(self._model.index(row, col)) == item + + +@pytest.fixture +def model_validator(qtmodeltester): + return ModelValidator(qtmodeltester) diff --git a/tests/unit/browser/test_history.py b/tests/unit/browser/test_history.py index 911184dc0..0b5e6eb41 100644 --- a/tests/unit/browser/test_history.py +++ b/tests/unit/browser/test_history.py @@ -21,7 +21,6 @@ import logging import os -import unittest.mock import pytest from PyQt5.QtCore import QUrl @@ -115,7 +114,7 @@ def test_clear(qtbot, tmpdir, hist, mocker): hist.add_url(QUrl('http://www.qutebrowser.org/')) m = mocker.patch('qutebrowser.browser.history.message.confirm_async', - new=unittest.mock.Mock, spec=[]) + new=mocker.Mock, spec=[]) hist.clear() assert m.called diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index 06c7ac1a6..3b1c1478a 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -45,26 +45,29 @@ from qutebrowser.commands import cmdexc [('foo', 'bar'), ('bar', 'foo'), ('bar', 'bar')], [('foo', 'bar'), ('bar', 'foo')]), ]) -def test_set_pattern(pattern, before, after, validate_model): +def test_set_pattern(pattern, before, after, model_validator): """Validate the filtering and sorting results of set_pattern.""" cat = listcategory.ListCategory('Foo', before) + model_validator.set_model(cat) cat.set_pattern(pattern) - validate_model(cat, after) + model_validator.validate(after) -def test_delete_cur_item(validate_model): +def test_delete_cur_item(model_validator): func = mock.Mock(spec=[]) cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')], delete_func=func) + model_validator.set_model(cat) idx = cat.index(0, 0) cat.delete_cur_item(idx) func.assert_called_once_with(['a', 'b']) - validate_model(cat, [('c', 'd')]) + model_validator.validate([('c', 'd')]) -def test_delete_cur_item_no_func(validate_model): +def test_delete_cur_item_no_func(model_validator): cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')]) + model_validator.set_model(cat) idx = cat.index(0, 0) with pytest.raises(cmdexc.CommandError, match="Cannot delete this item."): cat.delete_cur_item(idx) - validate_model(cat, [('a', 'b'), ('c', 'd')]) + model_validator.validate([('a', 'b'), ('c', 'd')]) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index c20f8f1c5..851bad257 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -60,14 +60,15 @@ pytestmark = pytest.mark.usefixtures('init_sql') [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), ]) -def test_sorting(sort_by, sort_order, data, expected, validate_model): +def test_sorting(sort_by, sort_order, data, expected, model_validator): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) + model_validator.set_model(cat) cat.set_pattern('') - validate_model(cat, expected) + model_validator.validate(expected) @pytest.mark.parametrize('pattern, filter_cols, before, after', [ @@ -115,23 +116,25 @@ def test_sorting(sort_by, sort_order, data, expected, validate_model): [("can't touch this", '', ''), ('a', '', '')], [("can't touch this", '', '')]), ]) -def test_set_pattern(pattern, filter_cols, before, after, validate_model): +def test_set_pattern(pattern, filter_cols, before, after, model_validator): """Validate the filtering and sorting results of set_pattern.""" table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) + model_validator.set_model(cat) cat.set_pattern(pattern) - validate_model(cat, after) + model_validator.validate(after) -def test_select(validate_model): +def test_select(model_validator): table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') + model_validator.set_model(cat) cat.set_pattern('') - validate_model(cat, [('bar', 'baz', 'foo')]) + model_validator.validate([('bar', 'baz', 'foo')]) def test_delete_cur_item(): From 182d067ff8f09de1f92461e5d87011055625eb1d Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 7 Jul 2017 21:16:50 -0400 Subject: [PATCH 153/161] SQL code review fixes. - Fix comment and empty line check in _parse_entry - connect layoutAboutToBeChanged signal - assert sort_order is None if sort_by is None - modify sql init failure message to ask about Qt sqlite support. --- qutebrowser/completion/models/completionmodel.py | 1 + qutebrowser/completion/models/sqlcategory.py | 2 ++ qutebrowser/misc/sql.py | 3 ++- tests/unit/completion/test_completionmodel.py | 9 ++++++--- tests/unit/completion/test_sqlcategory.py | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 778e0cac9..3e48076e5 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -59,6 +59,7 @@ class CompletionModel(QAbstractItemModel): def add_category(self, cat): """Add a completion category to the model.""" self._categories.append(cat) + cat.layoutAboutToBeChanged.connect(self.layoutAboutToBeChanged) cat.layoutChanged.connect(self.layoutChanged) def data(self, index, role=Qt.DisplayRole): diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 4819e940b..2664edbfe 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -59,6 +59,8 @@ class SqlCategory(QSqlQueryModel): if sort_by: assert sort_order in ['asc', 'desc'], sort_order querystr += ' order by {} {}'.format(sort_by, sort_order) + else: + assert sort_order is None, sort_order self._query = sql.Query(querystr, forward_only=False) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 4ae37f01f..a288df475 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -38,7 +38,8 @@ def init(db_path): """Initialize the SQL database connection.""" database = QSqlDatabase.addDatabase('QSQLITE') if not database.isValid(): - raise SqlException('Failed to add database. Is sqlite installed?') + raise SqlException('Failed to add database. ' + 'Are sqlite and Qt sqlite support installed?') database.setDatabaseName(db_path) if not database.open(): raise SqlException("Failed to open sqlite database at {}: {}" diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index ce0fe3765..4d1d3f123 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -36,7 +36,7 @@ def test_first_last_item(counts): """Test that first() and last() index to the first and last items.""" model = completionmodel.CompletionModel() for c in counts: - cat = mock.Mock(spec=['layoutChanged']) + cat = mock.Mock(spec=['layoutChanged', 'layoutAboutToBeChanged']) cat.rowCount = mock.Mock(return_value=c, spec=[]) model.add_category(cat) data = [i for i, rowCount in enumerate(counts) if rowCount > 0] @@ -60,7 +60,8 @@ def test_first_last_item(counts): def test_count(counts): model = completionmodel.CompletionModel() for c in counts: - cat = mock.Mock(spec=['rowCount', 'layoutChanged']) + cat = mock.Mock(spec=['rowCount', 'layoutChanged', + 'layoutAboutToBeChanged']) cat.rowCount = mock.Mock(return_value=c, spec=[]) model.add_category(cat) assert model.count() == sum(counts) @@ -70,7 +71,9 @@ def test_count(counts): def test_set_pattern(pat): """Validate the filtering and sorting results of set_pattern.""" model = completionmodel.CompletionModel() - cats = [mock.Mock(spec=['set_pattern', 'layoutChanged']) for _ in range(3)] + cats = [mock.Mock(spec=['set_pattern', 'layoutChanged', + 'layoutAboutToBeChanged']) + for _ in range(3)] for c in cats: c.set_pattern = mock.Mock(spec=[]) model.add_category(c) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 851bad257..a9321e33e 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -32,7 +32,7 @@ pytestmark = pytest.mark.usefixtures('init_sql') @pytest.mark.parametrize('sort_by, sort_order, data, expected', [ - (None, 'asc', + (None, None, [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), From ea459a1eca237955cc004da22145a0e6b598a6bf Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 12 Jul 2017 08:19:31 -0400 Subject: [PATCH 154/161] SQL code review fixes. - Ignore invalid variable name in flake8 (pylint already checks this and we don't want to have to double-ignore) - Fix and test completion bug with `:set asdf ` - Remove unused import - Use `assert not func.called` instead of `func.assert_not_called` for backwards compatibility --- .flake8 | 3 ++- qutebrowser/completion/completionwidget.py | 2 ++ tests/helpers/fixtures.py | 1 + tests/unit/browser/test_history.py | 1 - tests/unit/completion/test_completionwidget.py | 7 ++++++- 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.flake8 b/.flake8 index d967a505b..b87fef8b2 100644 --- a/.flake8 +++ b/.flake8 @@ -11,6 +11,7 @@ exclude = .*,__pycache__,resources.py # (for pytest's __tracebackhide__) # F401: Unused import # N802: function name should be lowercase +# N806: variable in function should be lowercase # P101: format string does contain unindexed parameters # P102: docstring does contain unindexed parameters # P103: other string does contain unindexed parameters @@ -27,7 +28,7 @@ exclude = .*,__pycache__,resources.py ignore = E128,E226,E265,E501,E402,E266,E722,E731, F401, - N802, + N802,N806 P101,P102,P103, D102,D103,D104,D105,D209,D211,D402,D403 min-version = 3.4.0 diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 9b8d31ad6..68466630a 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -290,6 +290,8 @@ class CompletionView(QTreeView): self.expand(model.index(i, 0)) def set_pattern(self, pattern): + if not self.model(): + return self.pattern = pattern with debug.log_time(log.completion, 'Set pattern {}'.format(pattern)): self.model().set_pattern(pattern) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index e38a3ce49..74c63f251 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -490,6 +490,7 @@ class ModelValidator: def __init__(self, modeltester): modeltester.data_display_may_return_none = True + self._model = None self._modeltester = modeltester def set_model(self, model): diff --git a/tests/unit/browser/test_history.py b/tests/unit/browser/test_history.py index 0b5e6eb41..5a8425646 100644 --- a/tests/unit/browser/test_history.py +++ b/tests/unit/browser/test_history.py @@ -20,7 +20,6 @@ """Tests for the global page history.""" import logging -import os import pytest from PyQt5.QtCore import QUrl diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index cad45b5c1..207e557a8 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -89,6 +89,11 @@ def test_set_pattern(completionview): model.set_pattern.assert_called_with('foo') +def test_set_pattern_no_model(completionview): + """Ensure that setting a pattern with no model does not fail.""" + completionview.set_pattern('foo') + + def test_maybe_update_geometry(completionview, config_stub, qtbot): """Ensure completion is resized only if shrink is True.""" with qtbot.assertNotEmitted(completionview.update_geometry): @@ -233,4 +238,4 @@ def test_completion_item_del_no_selection(completionview): completionview.set_model(model) with pytest.raises(cmdexc.CommandError, match='No item selected!'): completionview.completion_item_del() - func.assert_not_called() + assert not func.called From 1aed2470e5912264c8c90be58b97a729fd0e8bdf Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 12 Jul 2017 22:14:27 -0400 Subject: [PATCH 155/161] SQL code review. - Fix flake8 - history.clear should also clear completion table - call _resize_columns in set_model, not set_pattern - add more unit-testing for the history completion table --- .flake8 | 4 ++-- qutebrowser/browser/history.py | 1 + qutebrowser/completion/completionwidget.py | 3 ++- tests/unit/browser/test_history.py | 16 ++++++++++++++-- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.flake8 b/.flake8 index b87fef8b2..eada2c86d 100644 --- a/.flake8 +++ b/.flake8 @@ -28,7 +28,7 @@ exclude = .*,__pycache__,resources.py ignore = E128,E226,E265,E501,E402,E266,E722,E731, F401, - N802,N806 + N802, P101,P102,P103, D102,D103,D104,D105,D209,D211,D402,D403 min-version = 3.4.0 @@ -39,7 +39,7 @@ putty-ignore = /# pragma: no mccabe/ : +C901 tests/*/test_*.py : +D100,D101,D401 tests/conftest.py : +F403 - tests/unit/browser/webkit/test_history.py : +N806 + tests/unit/browser/test_history.py : +N806 tests/helpers/fixtures.py : +N806 tests/unit/browser/webkit/http/test_content_disposition.py : +D400 scripts/dev/ci/appveyor_install.py : +FI53 diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index b5b8af149..b9e791207 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -115,6 +115,7 @@ class WebHistory(sql.SqlTable): def _do_clear(self): self.delete_all() + self.completion.delete_all() def delete_url(self, url): """Remove all history entries with the given url. diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 68466630a..6e1e51680 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -286,16 +286,17 @@ class CompletionView(QTreeView): self._active = True self._maybe_show() + self._resize_columns() for i in range(model.rowCount()): self.expand(model.index(i, 0)) def set_pattern(self, pattern): + """Set the pattern on the underlying model.""" if not self.model(): return self.pattern = pattern with debug.log_time(log.completion, 'Set pattern {}'.format(pattern)): self.model().set_pattern(pattern) - self._resize_columns() self._maybe_update_geometry() self._maybe_show() diff --git a/tests/unit/browser/test_history.py b/tests/unit/browser/test_history.py index 5a8425646..81637f3d4 100644 --- a/tests/unit/browser/test_history.py +++ b/tests/unit/browser/test_history.py @@ -123,6 +123,7 @@ def test_clear_force(qtbot, tmpdir, hist): hist.add_url(QUrl('http://www.qutebrowser.org/')) hist.clear(force=True) assert not len(hist) + assert not len(hist.completion) def test_delete_url(hist): @@ -131,10 +132,16 @@ def test_delete_url(hist): hist.add_url(QUrl('http://example.com/2'), atime=0) before = set(hist) + completion_before = set(hist.completion) + hist.delete_url(QUrl('http://example.com/1')) + diff = before.difference(set(hist)) assert diff == {('http://example.com/1', '', 0, False)} + completion_diff = completion_before.difference(set(hist.completion)) + assert completion_diff == {('http://example.com/1', '', 0)} + @pytest.mark.parametrize('url, atime, title, redirect, expected_url', [ ('http://www.example.com', 12346, 'the title', False, @@ -146,15 +153,20 @@ def test_delete_url(hist): ('https://user:pass@example.com', 12346, 'the title', False, 'https://user@example.com'), ]) -def test_add_item(qtbot, hist, url, atime, title, redirect, expected_url): +def test_add_url(qtbot, hist, url, atime, title, redirect, expected_url): hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) assert list(hist) == [(expected_url, title, atime, redirect)] + if redirect: + assert not len(hist.completion) + else: + assert list(hist.completion) == [(expected_url, title, atime)] -def test_add_item_invalid(qtbot, hist, caplog): +def test_add_url_invalid(qtbot, hist, caplog): with caplog.at_level(logging.WARNING): hist.add_url(QUrl()) assert not list(hist) + assert not list(hist.completion) @pytest.mark.parametrize('level, url, req_url, expected', [ From 8745f80d90892c354bf82dc2058015a3b0e5156f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 13 Jul 2017 08:54:21 -0400 Subject: [PATCH 156/161] Fix qute://history SQL bug. The javascript history page was requesting the new start_time in ms, but the python code was expecting seconds. This is fixed by removing all the millisecond translations in the python code and only translating to milliseconds in the javascript code that formats dates. --- qutebrowser/browser/qutescheme.py | 6 ++++-- qutebrowser/javascript/history.js | 3 ++- tests/unit/browser/test_qutescheme.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 0e05cd7fb..0f86d0980 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -192,6 +192,8 @@ def history_data(start_time, offset=None): start_time: select history starting from this timestamp. offset: number of items to skip """ + # history atimes are stored as ints, ensure start_time is not a float + start_time = int(start_time) hist = objreg.get('web-history') if offset is not None: entries = hist.entries_before(start_time, limit=1000, offset=offset) @@ -200,7 +202,7 @@ def history_data(start_time, offset=None): end_time = start_time - 24*60*60 entries = hist.entries_between(end_time, start_time) - return [{"url": e.url, "title": e.title or e.url, "time": e.atime * 1000} + return [{"url": e.url, "title": e.title or e.url, "time": e.atime} for e in entries] @@ -252,7 +254,7 @@ def qute_history(url): start_time = time.mktime(next_date.timetuple()) - 1 history = [ (i["url"], i["title"], - datetime.datetime.fromtimestamp(i["time"]/1000), + datetime.datetime.fromtimestamp(i["time"]), QUrl(i["url"]).host()) for i in history_data(start_time) ] diff --git a/qutebrowser/javascript/history.js b/qutebrowser/javascript/history.js index 3f5f9def6..26b4405e9 100644 --- a/qutebrowser/javascript/history.js +++ b/qutebrowser/javascript/history.js @@ -174,7 +174,8 @@ window.loadHistory = (function() { for (var i = 0, len = history.length; i < len; i++) { var item = history[i]; - var currentItemDate = new Date(item.time); + // python's time.time returns seconds, but js Date expects ms + var currentItemDate = new Date(item.time * 1000); getSessionNode(currentItemDate).appendChild(makeHistoryRow( item.url, item.title, currentItemDate.toLocaleTimeString() )); diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 154af0355..693b2607c 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -129,8 +129,8 @@ class TestHistoryHandler: # test times end_time = start_time - 24*60*60 for item in items: - assert item['time'] <= start_time * 1000 - assert item['time'] > end_time * 1000 + assert item['time'] <= start_time + assert item['time'] > end_time def test_qute_history_benchmark(self, fake_web_history, benchmark, now): r = range(100000) From c32d452786508c9bc7f27f1e8c660001b310eaa9 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 14 Jul 2017 09:28:06 -0400 Subject: [PATCH 157/161] Add LIMIT to history query. For performance, re-introduce web-history-max-items. As the history query has now become a very specific multi-part query and history completion was the only consumer of SqlCategory, SqlCategory is now replaced by a HistoryCategory class. --- .../{sqlcategory.py => histcategory.py} | 72 ++++---- qutebrowser/completion/models/urlmodel.py | 13 +- qutebrowser/config/config.py | 1 - qutebrowser/config/configdata.py | 5 + tests/unit/completion/test_histcategory.py | 150 +++++++++++++++++ tests/unit/completion/test_models.py | 28 ++-- tests/unit/completion/test_sqlcategory.py | 158 ------------------ 7 files changed, 204 insertions(+), 223 deletions(-) rename qutebrowser/completion/models/{sqlcategory.py => histcategory.py} (54%) create mode 100644 tests/unit/completion/test_histcategory.py delete mode 100644 tests/unit/completion/test_sqlcategory.py diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/histcategory.py similarity index 54% rename from qutebrowser/completion/models/sqlcategory.py rename to qutebrowser/completion/models/histcategory.py index 2664edbfe..1899c2805 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""A completion model backed by SQL tables.""" +"""A completion category that queries the SQL History store.""" import re @@ -26,50 +26,50 @@ from PyQt5.QtSql import QSqlQueryModel from qutebrowser.misc import sql from qutebrowser.utils import debug from qutebrowser.commands import cmdexc +from qutebrowser.config import config -class SqlCategory(QSqlQueryModel): +class HistoryCategory(QSqlQueryModel): - """Wraps a SqlQuery for use as a completion category.""" + """A completion category that queries the SQL History store.""" - def __init__(self, name, *, title=None, filter_fields, sort_by=None, - sort_order=None, select='*', - delete_func=None, parent=None): - """Create a new completion category backed by a sql table. - - Args: - name: Name of the table in the database. - title: Title of category, defaults to table name. - filter_fields: Names of fields to apply filter pattern to. - select: A custom result column expression for the select statement. - sort_by: The name of the field to sort by, or None for no sorting. - sort_order: Either 'asc' or 'desc', if sort_by is non-None - delete_func: Callback to delete a selected item. - """ + def __init__(self, *, delete_func=None, parent=None): + """Create a new History completion category.""" super().__init__(parent=parent) - self.name = title or name + self.name = "History" - querystr = 'select {} from {} where ('.format(select, name) - # the incoming pattern will have literal % and _ escaped with '\' - # we need to tell sql to treat '\' as an escape character - querystr += ' or '.join("{} like :pattern escape '\\'".format(f) - for f in filter_fields) - querystr += ')' + # replace ' to avoid breaking the query + timefmt = "strftime('{}', last_atime, 'unixepoch')".format( + config.get('completion', 'timestamp-format').replace("'", "`")) - if sort_by: - assert sort_order in ['asc', 'desc'], sort_order - querystr += ' order by {} {}'.format(sort_by, sort_order) - else: - assert sort_order is None, sort_order + self._query = sql.Query(' '.join([ + "SELECT url, title, {}".format(timefmt), + "FROM CompletionHistory", + # the incoming pattern will have literal % and _ escaped with '\' + # we need to tell sql to treat '\' as an escape character + "WHERE (url LIKE :pat escape '\\' or title LIKE :pat escape '\\')", + self._atime_expr(), + "ORDER BY last_atime DESC", + ]), forward_only=False) - self._query = sql.Query(querystr, forward_only=False) - - # map filter_fields to indices - col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name)) - rec = col_query.run().record() - self.columns_to_filter = [rec.indexOf(n) for n in filter_fields] + # advertise that this model filters by URL and title + self.columns_to_filter = [0, 1] self.delete_func = delete_func + def _atime_expr(self): + """If max_items is set, return an expression to limit the query.""" + max_items = config.get('completion', 'web-history-max-items') + if max_items < 0: + return '' + + min_atime = sql.Query(' '.join([ + 'SELECT min(last_atime) FROM', + '(SELECT last_atime FROM CompletionHistory', + 'ORDER BY last_atime DESC LIMIT :limit)', + ])).run(limit=max_items).value() + + return "AND last_atime >= {}".format(min_atime) + def set_pattern(self, pattern): """Set the pattern used to filter results. @@ -83,7 +83,7 @@ class SqlCategory(QSqlQueryModel): pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) with debug.log_time('sql', 'Running completion query'): - self._query.run(pattern=pattern) + self._query.run(pat=pattern) self.setQuery(self._query) def delete_cur_item(self, index): diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 4f5fdeae9..0c5fbeacc 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -20,8 +20,7 @@ """Function to return the url completion model for the `open` command.""" from qutebrowser.completion.models import (completionmodel, listcategory, - sqlcategory) -from qutebrowser.config import config + histcategory) from qutebrowser.utils import log, objreg @@ -66,14 +65,6 @@ def url(): model.add_category(listcategory.ListCategory( 'Bookmarks', bookmarks, delete_func=_delete_bookmark)) - # replace 's to avoid breaking the query - timefmt = config.get('completion', 'timestamp-format').replace("'", "`") - select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) - hist_cat = sqlcategory.SqlCategory( - 'CompletionHistory', title='History', - sort_order='desc', sort_by='last_atime', - filter_fields=['url', 'title'], - select='url, title, {}'.format(select_time), - delete_func=_delete_history) + hist_cat = histcategory.HistoryCategory(delete_func=_delete_history) model.add_category(hist_cat) return model diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index 1245429c2..8bae2bae0 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -382,7 +382,6 @@ class ConfigManager(QObject): ('storage', 'offline-storage-default-quota'), ('storage', 'offline-web-application-cache-quota'), ('content', 'css-regions'), - ('completion', 'web-history-max-items'), ] CHANGED_OPTIONS = { ('content', 'cookies-accept'): diff --git a/qutebrowser/config/configdata.py b/qutebrowser/config/configdata.py index 6af04d9f4..23d3efb67 100644 --- a/qutebrowser/config/configdata.py +++ b/qutebrowser/config/configdata.py @@ -502,6 +502,11 @@ def data(readonly=False): "How many commands to save in the command history.\n\n" "0: no history / -1: unlimited"), + ('web-history-max-items', + SettingValue(typ.Int(minval=-1), '-1'), + "How many URLs to show in the web history.\n\n" + "0: no history / -1: unlimited"), + ('quick-complete', SettingValue(typ.Bool(), 'true'), "Whether to move on to the next part when there's only one " diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py new file mode 100644 index 000000000..f285c1dcc --- /dev/null +++ b/tests/unit/completion/test_histcategory.py @@ -0,0 +1,150 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Test the web history completion category.""" + +import unittest.mock +import time + +import pytest + +from qutebrowser.misc import sql +from qutebrowser.completion.models import histcategory +from qutebrowser.commands import cmdexc + + +@pytest.fixture +def hist(init_sql, config_stub): + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', + 'web-history-max-items': -1} + return sql.SqlTable('CompletionHistory', ['url', 'title', 'last_atime']) + + +@pytest.mark.parametrize('pattern, before, after', [ + ('foo', + [('foo', ''), ('bar', ''), ('aafobbb', '')], + [('foo',)]), + + ('FOO', + [('foo', ''), ('bar', ''), ('aafobbb', '')], + [('foo',)]), + + ('foo', + [('FOO', ''), ('BAR', ''), ('AAFOBBB', '')], + [('FOO',)]), + + ('foo', + [('baz', 'bar'), ('foo', ''), ('bar', 'foo')], + [('foo', ''), ('bar', 'foo')]), + + ('foo', + [('fooa', ''), ('foob', ''), ('fooc', '')], + [('fooa', ''), ('foob', ''), ('fooc', '')]), + + ('foo', + [('foo', 'bar'), ('bar', 'foo'), ('biz', 'baz')], + [('foo', 'bar'), ('bar', 'foo')]), + + ('foo bar', + [('foo', ''), ('bar foo', ''), ('xfooyybarz', '')], + [('xfooyybarz', '')]), + + ('foo%bar', + [('foo%bar', ''), ('foo bar', ''), ('foobar', '')], + [('foo%bar', '')]), + + ('_', + [('a_b', ''), ('__a', ''), ('abc', '')], + [('a_b', ''), ('__a', '')]), + + ('%', + [('\\foo', '\\bar')], + []), + + ("can't", + [("can't touch this", ''), ('a', '')], + [("can't touch this", '')]), +]) +def test_set_pattern(pattern, before, after, model_validator, hist): + """Validate the filtering and sorting results of set_pattern.""" + for row in before: + hist.insert({'url': row[0], 'title': row[1], 'last_atime': 1}) + cat = histcategory.HistoryCategory() + model_validator.set_model(cat) + cat.set_pattern(pattern) + model_validator.validate(after) + + +@pytest.mark.parametrize('max_items, before, after', [ + (-1, [ + ('a', 'a', '2017-04-16'), + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ], [ + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ('a', 'a', '2017-04-16'), + ]), + (3, [ + ('a', 'a', '2017-04-16'), + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ], [ + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ('a', 'a', '2017-04-16'), + ]), + (2, [ + ('a', 'a', '2017-04-16'), + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ], [ + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ]) +]) +def test_sorting(max_items, before, after, model_validator, hist, config_stub): + """Validate the filtering and sorting results of set_pattern.""" + config_stub.data['completion']['web-history-max-items'] = max_items + for url, title, atime in before: + timestamp = time.mktime(time.strptime(atime, '%Y-%m-%d')) + hist.insert({'url': url, 'title': title, 'last_atime': timestamp}) + cat = histcategory.HistoryCategory() + model_validator.set_model(cat) + cat.set_pattern('') + model_validator.validate(after) + + +def test_delete_cur_item(hist): + hist.insert({'url': 'foo', 'title': 'Foo'}) + hist.insert({'url': 'bar', 'title': 'Bar'}) + func = unittest.mock.Mock(spec=[]) + cat = histcategory.HistoryCategory(delete_func=func) + cat.set_pattern('') + cat.delete_cur_item(cat.index(0, 0)) + func.assert_called_with(['foo', 'Foo', '']) + + +def test_delete_cur_item_no_func(hist): + hist.insert({'url': 'foo', 'title': 1}) + hist.insert({'url': 'bar', 'title': 2}) + cat = histcategory.HistoryCategory() + cat.set_pattern('') + with pytest.raises(cmdexc.CommandError, match='Cannot delete this item'): + cat.delete_cur_item(cat.index(0, 0)) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 4fab0c402..5b632eb3a 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -137,8 +137,10 @@ def bookmarks(bookmark_manager_stub): @pytest.fixture -def web_history(init_sql, stubs): +def web_history(init_sql, stubs, config_stub): """Fixture which provides a web-history object.""" + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', + 'web-history-max-items': -1} stub = history.WebHistory() objreg.register('web-history', stub) yield stub @@ -266,7 +268,7 @@ def test_bookmark_completion(qtmodeltester, bookmarks): }) -def test_url_completion(qtmodeltester, config_stub, web_history_populated, +def test_url_completion(qtmodeltester, web_history_populated, quickmarks, bookmarks): """Test the results of url completion. @@ -275,7 +277,6 @@ def test_url_completion(qtmodeltester, config_stub, web_history_populated, - entries are sorted by access time - only the most recent entry is included for each url """ - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() model.set_pattern('') qtmodeltester.data_display_may_return_none = True @@ -318,11 +319,10 @@ def test_url_completion(qtmodeltester, config_stub, web_history_populated, ('foo%bar', '', '%', 1), ('foobar', '', '%', 0), ]) -def test_url_completion_pattern(config_stub, web_history, - quickmark_manager_stub, bookmark_manager_stub, - url, title, pattern, rowcount): +def test_url_completion_pattern(web_history, quickmark_manager_stub, + bookmark_manager_stub, url, title, pattern, + rowcount): """Test that url completion filters by url and title.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} web_history.add_url(QUrl(url), title) model = urlmodel.url() model.set_pattern(pattern) @@ -330,10 +330,9 @@ def test_url_completion_pattern(config_stub, web_history, assert model.rowCount(model.index(2, 0)) == rowcount -def test_url_completion_delete_bookmark(qtmodeltester, config_stub, bookmarks, +def test_url_completion_delete_bookmark(qtmodeltester, bookmarks, web_history, quickmarks): """Test deleting a bookmark from the url completion model.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() model.set_pattern('') qtmodeltester.data_display_may_return_none = True @@ -353,11 +352,10 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, bookmarks, assert len_before == len(bookmarks.marks) + 1 -def test_url_completion_delete_quickmark(qtmodeltester, config_stub, +def test_url_completion_delete_quickmark(qtmodeltester, quickmarks, web_history, bookmarks, qtbot): """Test deleting a bookmark from the url completion model.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() model.set_pattern('') qtmodeltester.data_display_may_return_none = True @@ -377,11 +375,10 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, assert len_before == len(quickmarks.marks) + 1 -def test_url_completion_delete_history(qtmodeltester, config_stub, +def test_url_completion_delete_history(qtmodeltester, web_history_populated, quickmarks, bookmarks): """Test deleting a history entry.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() model.set_pattern('') qtmodeltester.data_display_may_return_none = True @@ -598,14 +595,11 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, }) -def test_url_completion_benchmark(benchmark, config_stub, +def test_url_completion_benchmark(benchmark, quickmark_manager_stub, bookmark_manager_stub, web_history): """Benchmark url completion.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - 'web-history-max-items': 1000} - r = range(100000) entries = { 'last_atime': list(r), diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py deleted file mode 100644 index a9321e33e..000000000 --- a/tests/unit/completion/test_sqlcategory.py +++ /dev/null @@ -1,158 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Test SQL-based completions.""" - -import unittest.mock - -import pytest - -from qutebrowser.misc import sql -from qutebrowser.completion.models import sqlcategory -from qutebrowser.commands import cmdexc - - -pytestmark = pytest.mark.usefixtures('init_sql') - - -@pytest.mark.parametrize('sort_by, sort_order, data, expected', [ - (None, None, - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), - - ('a', 'asc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - - ('a', 'desc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - - ('b', 'asc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - - ('b', 'desc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - - ('c', 'asc', - [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], - [('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]), - - ('c', 'desc', - [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], - [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), -]) -def test_sorting(sort_by, sort_order, data, expected, model_validator): - table = sql.SqlTable('Foo', ['a', 'b', 'c']) - for row in data: - table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, - sort_order=sort_order) - model_validator.set_model(cat) - cat.set_pattern('') - model_validator.validate(expected) - - -@pytest.mark.parametrize('pattern, filter_cols, before, after', [ - ('foo', [0], - [('foo', '', ''), ('bar', '', ''), ('aafobbb', '', '')], - [('foo',)]), - - ('foo', [0], - [('baz', 'bar', 'foo'), ('foo', '', ''), ('bar', 'foo', '')], - [('foo', '', '')]), - - ('foo', [0], - [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')], - [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]), - - ('foo', [1], - [('foo', 'bar', ''), ('bar', 'foo', '')], - [('bar', 'foo', '')]), - - ('foo', [0, 1], - [('foo', 'bar', ''), ('bar', 'foo', ''), ('biz', 'baz', 'foo')], - [('foo', 'bar', ''), ('bar', 'foo', '')]), - - ('foo', [0, 1, 2], - [('foo', '', ''), ('bar', '', ''), ('baz', 'bar', 'foo')], - [('foo', '', ''), ('baz', 'bar', 'foo')]), - - ('foo bar', [0], - [('foo', '', ''), ('bar foo', '', ''), ('xfooyybarz', '', '')], - [('xfooyybarz', '', '')]), - - ('foo%bar', [0], - [('foo%bar', '', ''), ('foo bar', '', ''), ('foobar', '', '')], - [('foo%bar', '', '')]), - - ('_', [0], - [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')], - [('a_b', '', ''), ('__a', '', '')]), - - ('%', [0, 1], - [('\\foo', '\\bar', '')], - []), - - ("can't", [0], - [("can't touch this", '', ''), ('a', '', '')], - [("can't touch this", '', '')]), -]) -def test_set_pattern(pattern, filter_cols, before, after, model_validator): - """Validate the filtering and sorting results of set_pattern.""" - table = sql.SqlTable('Foo', ['a', 'b', 'c']) - for row in before: - table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) - filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] - cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) - model_validator.set_model(cat) - cat.set_pattern(pattern) - model_validator.validate(after) - - -def test_select(model_validator): - table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') - model_validator.set_model(cat) - cat.set_pattern('') - model_validator.validate([('bar', 'baz', 'foo')]) - - -def test_delete_cur_item(): - table = sql.SqlTable('Foo', ['a', 'b']) - table.insert({'a': 'foo', 'b': 1}) - table.insert({'a': 'bar', 'b': 2}) - func = unittest.mock.Mock(spec=[]) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], delete_func=func) - cat.set_pattern('') - cat.delete_cur_item(cat.index(0, 0)) - func.assert_called_with(['foo', 1]) - - -def test_delete_cur_item_no_func(): - table = sql.SqlTable('Foo', ['a', 'b']) - table.insert({'a': 'foo', 'b': 1}) - table.insert({'a': 'bar', 'b': 2}) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a']) - cat.set_pattern('') - with pytest.raises(cmdexc.CommandError, match='Cannot delete this item'): - cat.delete_cur_item(cat.index(0, 0)) From f45acaa9c8104a865b2d5790f6e26ff34cbee77a Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 17 Jul 2017 08:37:24 -0400 Subject: [PATCH 158/161] Fix coverage check for sqlcategory rename. --- scripts/dev/check_coverage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 950f21ff1..e63bb821f 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -156,8 +156,8 @@ PERFECT_FILES = [ ('tests/unit/completion/test_models.py', 'completion/models/urlmodel.py'), - ('tests/unit/completion/test_sqlcategory.py', - 'completion/models/sqlcategory.py'), + ('tests/unit/completion/test_histcategory.py', + 'completion/models/histcategory.py'), ] From 0eb347186cf0174166d6452a2420d20b511cecff Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 20 Jul 2017 08:59:12 -0400 Subject: [PATCH 159/161] Add 'localtime' to sql history query. We need to tell sqlite to convert the timestamps to localtime during formatting, otherwise it formats them as though you are in UTC. Also fix up a few uses of mktime. --- qutebrowser/browser/qutescheme.py | 2 +- qutebrowser/completion/models/histcategory.py | 7 ++++--- tests/unit/completion/test_histcategory.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 0f86d0980..e712adcf8 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -251,7 +251,7 @@ def qute_history(url): prev_date = curr_date - one_day # start_time is the last second of curr_date - start_time = time.mktime(next_date.timetuple()) - 1 + start_time = next_date.timestamp() - 1 history = [ (i["url"], i["title"], datetime.datetime.fromtimestamp(i["time"]), diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index 1899c2805..fa8443a60 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -38,9 +38,10 @@ class HistoryCategory(QSqlQueryModel): super().__init__(parent=parent) self.name = "History" - # replace ' to avoid breaking the query - timefmt = "strftime('{}', last_atime, 'unixepoch')".format( - config.get('completion', 'timestamp-format').replace("'", "`")) + # replace ' in timestamp-format to avoid breaking the query + timefmt = ("strftime('{}', last_atime, 'unixepoch', 'localtime')" + .format(config.get('completion', 'timestamp-format') + .replace("'", "`"))) self._query = sql.Query(' '.join([ "SELECT url, title, {}".format(timefmt), diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index f285c1dcc..0b5fcb915 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -20,7 +20,7 @@ """Test the web history completion category.""" import unittest.mock -import time +import datetime import pytest @@ -123,7 +123,7 @@ def test_sorting(max_items, before, after, model_validator, hist, config_stub): """Validate the filtering and sorting results of set_pattern.""" config_stub.data['completion']['web-history-max-items'] = max_items for url, title, atime in before: - timestamp = time.mktime(time.strptime(atime, '%Y-%m-%d')) + timestamp = datetime.datetime.strptime(atime, '%Y-%m-%d').timestamp() hist.insert({'url': url, 'title': title, 'last_atime': timestamp}) cat = histcategory.HistoryCategory() model_validator.set_model(cat) From 1175543ce16c4f9e4925148307ca5a87ac69983e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 20 Jul 2017 22:07:37 -0400 Subject: [PATCH 160/161] Fix qutescheme timestamp error. A date object doesn't have a timestamp property. Go back to using mktime. --- qutebrowser/browser/qutescheme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index e712adcf8..0f86d0980 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -251,7 +251,7 @@ def qute_history(url): prev_date = curr_date - one_day # start_time is the last second of curr_date - start_time = next_date.timestamp() - 1 + start_time = time.mktime(next_date.timetuple()) - 1 history = [ (i["url"], i["title"], datetime.datetime.fromtimestamp(i["time"]), From 33a9c8cce6082ba1a158faaa5b409133dddf644a Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 21 Jul 2017 07:59:22 -0400 Subject: [PATCH 161/161] Add listcategory to perfect_files. --- scripts/dev/check_coverage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index e63bb821f..c9e3836b6 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -158,6 +158,8 @@ PERFECT_FILES = [ 'completion/models/urlmodel.py'), ('tests/unit/completion/test_histcategory.py', 'completion/models/histcategory.py'), + ('tests/unit/completion/test_listcategory.py', + 'completion/models/listcategory.py'), ]