From 08bb3f4f19b5117a00a7756119c4666d2065cfab Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 16 Sep 2016 15:41:54 -0400 Subject: [PATCH 001/337] Implement completion models as functions. First step of Completion Model/View revamping (#74). Rewrite the completion models as functions that each return an instance of a CompletionModel class. Caching is removed from all models except the UrlModel. Models other than the UrlModel can be generated very quickly so caching just adds needless complexity and can lead to incorrect results if one forgets to wire up a signal. --- qutebrowser/completion/completer.py | 17 +- qutebrowser/completion/completionwidget.py | 5 +- qutebrowser/completion/models/base.py | 25 +- qutebrowser/completion/models/configmodel.py | 179 +++------- qutebrowser/completion/models/instances.py | 193 ++--------- qutebrowser/completion/models/miscmodels.py | 293 +++++----------- qutebrowser/completion/models/sortfilter.py | 2 +- qutebrowser/completion/models/urlmodel.py | 324 +++++++++--------- tests/unit/completion/test_column_widths.py | 50 --- tests/unit/completion/test_completer.py | 104 +++--- .../unit/completion/test_completionwidget.py | 10 +- tests/unit/completion/test_models.py | 33 +- tests/unit/completion/test_sortfilter.py | 10 +- 13 files changed, 431 insertions(+), 814 deletions(-) delete mode 100644 tests/unit/completion/test_column_widths.py diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 96d937829..21af74da5 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -72,20 +72,7 @@ class Completer(QObject): Return: A completion model or None. """ - if completion == usertypes.Completion.option: - section = pos_args[0] - model = instances.get(completion).get(section) - elif completion == usertypes.Completion.value: - section = pos_args[0] - option = pos_args[1] - try: - model = instances.get(completion)[section][option] - except KeyError: - # No completion model for this section/option. - model = None - else: - model = instances.get(completion) - + model = instances.get(completion)(*pos_args) if model is None: return None else: @@ -109,7 +96,7 @@ class Completer(QObject): log.completion.debug("After removing flags: {}".format(before_cursor)) if not before_cursor: # '|' or 'set|' - model = instances.get(usertypes.Completion.command) + model = instances.get(usertypes.Completion.command)() return sortfilter.CompletionFilterModel(source=model, parent=self) try: cmd = cmdutils.cmd_dict[before_cursor[0]] diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 490fcd6c0..f12027340 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -28,7 +28,6 @@ from PyQt5.QtCore import pyqtSlot, pyqtSignal, Qt, QItemSelectionModel, QSize from qutebrowser.config import config, style from qutebrowser.completion import completiondelegate -from qutebrowser.completion.models import base from qutebrowser.utils import utils, usertypes, objreg from qutebrowser.commands import cmdexc, cmdutils @@ -112,7 +111,7 @@ class CompletionView(QTreeView): # objreg.get('config').changed.connect(self.init_command_completion) objreg.get('config').changed.connect(self._on_config_changed) - self._column_widths = base.BaseCompletionModel.COLUMN_WIDTHS + self._column_widths = (30, 70, 0) self._active = False self._delegate = completiondelegate.CompletionItemDelegate(self) @@ -300,7 +299,7 @@ class CompletionView(QTreeView): if pattern is not None: model.set_pattern(pattern) - self._column_widths = model.srcmodel.COLUMN_WIDTHS + self._column_widths = model.srcmodel.column_widths self._resize_columns() self._maybe_update_geometry() diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py index b1cad276a..43f3a1b48 100644 --- a/qutebrowser/completion/models/base.py +++ b/qutebrowser/completion/models/base.py @@ -33,26 +33,27 @@ Role = usertypes.enum('Role', ['sort', 'userdata'], start=Qt.UserRole, is_int=True) -class BaseCompletionModel(QStandardItemModel): +class CompletionModel(QStandardItemModel): """A simple QStandardItemModel adopted for completions. Used for showing completions later in the CompletionView. Supports setting marks and adding new categories/items easily. - Class Attributes: - COLUMN_WIDTHS: The width percentages of the columns used in the - completion view. - DUMB_SORT: the dumb sorting used by the model + Attributes: + column_widths: The width percentages of the columns used in the + completion view. + dumb_sort: the dumb sorting used by the model """ - COLUMN_WIDTHS = (30, 70, 0) - DUMB_SORT = None - - def __init__(self, parent=None): + def __init__(self, dumb_sort=None, column_widths=(30, 70, 0), + columns_to_filter=None, delete_cur_item=None, parent=None): super().__init__(parent) self.setColumnCount(3) - self.columns_to_filter = [0] + self.columns_to_filter = columns_to_filter or [0] + self.dumb_sort = dumb_sort + self.column_widths = column_widths + self.delete_cur_item = delete_cur_item def new_category(self, name, sort=None): """Add a new category to the model. @@ -103,10 +104,6 @@ class BaseCompletionModel(QStandardItemModel): nameitem.setData(userdata, Role.userdata) return nameitem, descitem, miscitem - def delete_cur_item(self, completion): - """Delete the selected item.""" - raise NotImplementedError - def flags(self, index): """Return the item flags for index. diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index c9e9850d1..5ed2a47d0 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -17,142 +17,75 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""CompletionModels for the config.""" +"""Functions that return config-related completion models.""" -from PyQt5.QtCore import pyqtSlot, Qt - -from qutebrowser.config import config, configdata -from qutebrowser.utils import log, qtutils, objreg +from qutebrowser.config import config, configdata, configexc from qutebrowser.completion.models import base -class SettingSectionCompletionModel(base.BaseCompletionModel): - +def section(): """A CompletionModel filled with settings sections.""" - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - COLUMN_WIDTHS = (20, 70, 10) - - def __init__(self, parent=None): - super().__init__(parent) - cat = self.new_category("Sections") - for name in configdata.DATA: - desc = configdata.SECTION_DESC[name].splitlines()[0].strip() - self.new_item(cat, name, desc) + model = base.CompletionModel(column_widths=(20, 70, 10)) + cat = model.new_category("Sections") + for name in configdata.DATA: + desc = configdata.SECTION_DESC[name].splitlines()[0].strip() + model.new_item(cat, name, desc) + return model -class SettingOptionCompletionModel(base.BaseCompletionModel): - +def option(sectname): """A CompletionModel filled with settings and their descriptions. - Attributes: - _misc_items: A dict of the misc. column items which will be set later. - _section: The config section this model shows. + Args: + sectname: The name of the config section this model shows. """ - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - COLUMN_WIDTHS = (20, 70, 10) - - def __init__(self, section, parent=None): - super().__init__(parent) - cat = self.new_category(section) - sectdata = configdata.DATA[section] - self._misc_items = {} - self._section = section - objreg.get('config').changed.connect(self.update_misc_column) - for name in sectdata: - try: - desc = sectdata.descriptions[name] - except (KeyError, AttributeError): - # Some stuff (especially ValueList items) don't have a - # description. - desc = "" - else: - desc = desc.splitlines()[0] - value = config.get(section, name, raw=True) - _valitem, _descitem, miscitem = self.new_item(cat, name, desc, - value) - self._misc_items[name] = miscitem - - @pyqtSlot(str, str) - def update_misc_column(self, section, option): - """Update misc column when config changed.""" - if section != self._section: - return + model = base.CompletionModel(column_widths=(20, 70, 10)) + cat = model.new_category(sectname) + try: + sectdata = configdata.DATA[sectname] + except KeyError: + return None + for name in sectdata: try: - item = self._misc_items[option] - except KeyError: - log.completion.debug("Couldn't get item {}.{} from model!".format( - section, option)) - # changed before init - return - val = config.get(section, option, raw=True) - idx = item.index() - qtutils.ensure_valid(idx) - ok = self.setData(idx, val, Qt.DisplayRole) - if not ok: - raise ValueError("Setting data failed! (section: {}, option: {}, " - "value: {})".format(section, option, val)) + desc = sectdata.descriptions[name] + except (KeyError, AttributeError): + # Some stuff (especially ValueList items) don't have a + # description. + desc = "" + else: + desc = desc.splitlines()[0] + val = config.get(sectname, name, raw=True) + model.new_item(cat, name, desc, val) + return model -class SettingValueCompletionModel(base.BaseCompletionModel): - +def value(sectname, optname): """A CompletionModel filled with setting values. - Attributes: - _section: The config section this model shows. - _option: The config option this model shows. + Args: + sectname: The name of the config section this model shows. + optname: The name of the config option this model shows. """ - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - COLUMN_WIDTHS = (20, 70, 10) - - def __init__(self, section, option, parent=None): - super().__init__(parent) - self._section = section - self._option = option - objreg.get('config').changed.connect(self.update_current_value) - cur_cat = self.new_category("Current/Default", sort=0) - value = config.get(section, option, raw=True) - if not value: - value = '""' - self.cur_item, _descitem, _miscitem = self.new_item(cur_cat, value, - "Current value") - default_value = configdata.DATA[section][option].default() - if not default_value: - default_value = '""' - self.new_item(cur_cat, default_value, "Default value") - if hasattr(configdata.DATA[section], 'valtype'): - # Same type for all values (ValueList) - vals = configdata.DATA[section].valtype.complete() - else: - if option is None: - raise ValueError("option may only be None for ValueList " - "sections, but {} is not!".format(section)) - # Different type for each value (KeyValue) - vals = configdata.DATA[section][option].typ.complete() - if vals is not None: - cat = self.new_category("Completions", sort=1) - for (val, desc) in vals: - self.new_item(cat, val, desc) - - @pyqtSlot(str, str) - def update_current_value(self, section, option): - """Update current value when config changed.""" - if (section, option) != (self._section, self._option): - return - value = config.get(section, option, raw=True) - if not value: - value = '""' - idx = self.cur_item.index() - qtutils.ensure_valid(idx) - ok = self.setData(idx, value, Qt.DisplayRole) - if not ok: - raise ValueError("Setting data failed! (section: {}, option: {}, " - "value: {})".format(section, option, value)) + model = base.CompletionModel(column_widths=(20, 70, 10)) + cur_cat = model.new_category("Current/Default", sort=0) + try: + val = config.get(sectname, optname, raw=True) or '""' + except (configexc.NoSectionError, configexc.NoOptionError): + return None + model.new_item(cur_cat, val, "Current value") + default_value = configdata.DATA[sectname][optname].default() or '""' + model.new_item(cur_cat, default_value, "Default value") + if hasattr(configdata.DATA[sectname], 'valtype'): + # Same type for all values (ValueList) + vals = configdata.DATA[sectname].valtype.complete() + else: + if optname is None: + raise ValueError("optname may only be None for ValueList " + "sections, but {} is not!".format(sectname)) + # Different type for each value (KeyValue) + vals = configdata.DATA[sectname][optname].typ.complete() + if vals is not None: + cat = model.new_category("Completions", sort=1) + for (val, desc) in vals: + model.new_item(cat, val, desc) + return model diff --git a/qutebrowser/completion/models/instances.py b/qutebrowser/completion/models/instances.py index f7eaaca86..ce109ae7a 100644 --- a/qutebrowser/completion/models/instances.py +++ b/qutebrowser/completion/models/instances.py @@ -17,180 +17,37 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""Global instances of the completion models. - -Module attributes: - _instances: A dict of available completions. - INITIALIZERS: A {usertypes.Completion: callable} dict of functions to - initialize completions. -""" - -import functools +"""Global instances of the completion models.""" +from qutebrowser.utils import usertypes from qutebrowser.completion.models import miscmodels, urlmodel, configmodel -from qutebrowser.utils import objreg, usertypes, log, debug -from qutebrowser.config import configdata, config - - -_instances = {} - - -def _init_command_completion(): - """Initialize the command completion model.""" - log.completion.debug("Initializing command completion.") - model = miscmodels.CommandCompletionModel() - _instances[usertypes.Completion.command] = model - - -def _init_helptopic_completion(): - """Initialize the helptopic completion model.""" - log.completion.debug("Initializing helptopic completion.") - model = miscmodels.HelpCompletionModel() - _instances[usertypes.Completion.helptopic] = model - - -def _init_url_completion(): - """Initialize the URL completion model.""" - log.completion.debug("Initializing URL completion.") - with debug.log_time(log.completion, 'URL completion init'): - model = urlmodel.UrlCompletionModel() - _instances[usertypes.Completion.url] = model - - -def _init_tab_completion(): - """Initialize the tab completion model.""" - log.completion.debug("Initializing tab completion.") - with debug.log_time(log.completion, 'tab completion init'): - model = miscmodels.TabCompletionModel() - _instances[usertypes.Completion.tab] = model - - -def _init_setting_completions(): - """Initialize setting completion models.""" - log.completion.debug("Initializing setting completion.") - _instances[usertypes.Completion.section] = ( - configmodel.SettingSectionCompletionModel()) - _instances[usertypes.Completion.option] = {} - _instances[usertypes.Completion.value] = {} - for sectname in configdata.DATA: - opt_model = configmodel.SettingOptionCompletionModel(sectname) - _instances[usertypes.Completion.option][sectname] = opt_model - _instances[usertypes.Completion.value][sectname] = {} - for opt in configdata.DATA[sectname]: - val_model = configmodel.SettingValueCompletionModel(sectname, opt) - _instances[usertypes.Completion.value][sectname][opt] = val_model - - -def init_quickmark_completions(): - """Initialize quickmark completion models.""" - log.completion.debug("Initializing quickmark completion.") - try: - _instances[usertypes.Completion.quickmark_by_name].deleteLater() - except KeyError: - pass - model = miscmodels.QuickmarkCompletionModel() - _instances[usertypes.Completion.quickmark_by_name] = model - - -def init_bookmark_completions(): - """Initialize bookmark completion models.""" - log.completion.debug("Initializing bookmark completion.") - try: - _instances[usertypes.Completion.bookmark_by_url].deleteLater() - except KeyError: - pass - model = miscmodels.BookmarkCompletionModel() - _instances[usertypes.Completion.bookmark_by_url] = model - - -def init_session_completion(): - """Initialize session completion model.""" - log.completion.debug("Initializing session completion.") - try: - _instances[usertypes.Completion.sessions].deleteLater() - except KeyError: - pass - model = miscmodels.SessionCompletionModel() - _instances[usertypes.Completion.sessions] = model - - -def _init_bind_completion(): - """Initialize the command completion model.""" - log.completion.debug("Initializing bind completion.") - model = miscmodels.BindCompletionModel() - _instances[usertypes.Completion.bind] = model - - -INITIALIZERS = { - usertypes.Completion.command: _init_command_completion, - usertypes.Completion.helptopic: _init_helptopic_completion, - usertypes.Completion.url: _init_url_completion, - usertypes.Completion.tab: _init_tab_completion, - usertypes.Completion.section: _init_setting_completions, - usertypes.Completion.option: _init_setting_completions, - usertypes.Completion.value: _init_setting_completions, - usertypes.Completion.quickmark_by_name: init_quickmark_completions, - usertypes.Completion.bookmark_by_url: init_bookmark_completions, - usertypes.Completion.sessions: init_session_completion, - usertypes.Completion.bind: _init_bind_completion, -} def get(completion): """Get a certain completion. Initializes the completion if needed.""" - try: - return _instances[completion] - except KeyError: - if completion in INITIALIZERS: - INITIALIZERS[completion]() - return _instances[completion] - else: - raise - - -def update(completions): - """Update an already existing completion. - - Args: - completions: An iterable of usertypes.Completions. - """ - did_run = [] - for completion in completions: - if completion in _instances: - func = INITIALIZERS[completion] - if func not in did_run: - func() - did_run.append(func) - - -@config.change_filter('aliases', function=True) -def _update_aliases(): - """Update completions that include command aliases.""" - update([usertypes.Completion.command]) + if completion == usertypes.Completion.command: + return miscmodels.command + if completion == usertypes.Completion.helptopic: + return miscmodels.helptopic + if completion == usertypes.Completion.tab: + return miscmodels.buffer + if completion == usertypes.Completion.quickmark_by_name: + return miscmodels.quickmark + if completion == usertypes.Completion.bookmark_by_url: + return miscmodels.bookmark + if completion == usertypes.Completion.sessions: + return miscmodels.session + if completion == usertypes.Completion.bind: + return miscmodels.bind + if completion == usertypes.Completion.section: + return configmodel.section + if completion == usertypes.Completion.option: + return configmodel.option + if completion == usertypes.Completion.value: + return configmodel.value + if completion == usertypes.Completion.url: + return urlmodel.url def init(): - """Initialize completions. Note this only connects signals.""" - quickmark_manager = objreg.get('quickmark-manager') - quickmark_manager.changed.connect( - functools.partial(update, [usertypes.Completion.quickmark_by_name])) - - bookmark_manager = objreg.get('bookmark-manager') - bookmark_manager.changed.connect( - functools.partial(update, [usertypes.Completion.bookmark_by_url])) - - session_manager = objreg.get('session-manager') - session_manager.update_completion.connect( - functools.partial(update, [usertypes.Completion.sessions])) - - history = objreg.get('web-history') - history.async_read_done.connect( - functools.partial(update, [usertypes.Completion.url])) - - keyconf = objreg.get('key-config') - keyconf.changed.connect( - functools.partial(update, [usertypes.Completion.command])) - keyconf.changed.connect( - functools.partial(update, [usertypes.Completion.bind])) - - objreg.get('config').changed.connect(_update_aliases) + pass diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 5ab381c43..bcbb94177 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -17,252 +17,139 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""Misc. CompletionModels.""" +"""Functions that return miscellaneous completion models.""" -from PyQt5.QtCore import Qt, QTimer, pyqtSlot +from PyQt5.QtCore import Qt -from qutebrowser.browser import browsertab from qutebrowser.config import config, configdata from qutebrowser.utils import objreg, log, qtutils from qutebrowser.commands import cmdutils from qutebrowser.completion.models import base -class CommandCompletionModel(base.BaseCompletionModel): - +def command(): """A CompletionModel filled with non-hidden commands and descriptions.""" - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - COLUMN_WIDTHS = (20, 60, 20) - - def __init__(self, parent=None): - super().__init__(parent) - cmdlist = _get_cmd_completions(include_aliases=True, - include_hidden=False) - cat = self.new_category("Commands") - for (name, desc, misc) in cmdlist: - self.new_item(cat, name, desc, misc) + model = base.CompletionModel(column_widths=(20, 60, 20)) + cmdlist = _get_cmd_completions(include_aliases=True, include_hidden=False) + cat = model.new_category("Commands") + for (name, desc, misc) in cmdlist: + model.new_item(cat, name, desc, misc) + return model -class HelpCompletionModel(base.BaseCompletionModel): - +def helptopic(): """A CompletionModel filled with help topics.""" + model = base.CompletionModel() - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method + cmdlist = _get_cmd_completions(include_aliases=False, include_hidden=True, + prefix=':') + cat = model.new_category("Commands") + for (name, desc, misc) in cmdlist: + model.new_item(cat, name, desc, misc) - COLUMN_WIDTHS = (20, 60, 20) - - def __init__(self, parent=None): - super().__init__(parent) - self._init_commands() - self._init_settings() - - def _init_commands(self): - """Fill completion with :command entries.""" - cmdlist = _get_cmd_completions(include_aliases=False, - include_hidden=True, prefix=':') - cat = self.new_category("Commands") - for (name, desc, misc) in cmdlist: - self.new_item(cat, name, desc, misc) - - def _init_settings(self): - """Fill completion with section->option entries.""" - cat = self.new_category("Settings") - for sectname, sectdata in configdata.DATA.items(): - for optname in sectdata: - try: - desc = sectdata.descriptions[optname] - except (KeyError, AttributeError): - # Some stuff (especially ValueList items) don't have a - # description. - desc = "" - else: - desc = desc.splitlines()[0] - name = '{}->{}'.format(sectname, optname) - self.new_item(cat, name, desc) + cat = model.new_category("Settings") + for sectname, sectdata in configdata.DATA.items(): + for optname in sectdata: + try: + desc = sectdata.descriptions[optname] + except (KeyError, AttributeError): + # Some stuff (especially ValueList items) don't have a + # description. + desc = "" + else: + desc = desc.splitlines()[0] + name = '{}->{}'.format(sectname, optname) + model.new_item(cat, name, desc) + return model -class QuickmarkCompletionModel(base.BaseCompletionModel): - +def quickmark(): """A CompletionModel filled with all quickmarks.""" - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - def __init__(self, parent=None): - super().__init__(parent) - cat = self.new_category("Quickmarks") - quickmarks = objreg.get('quickmark-manager').marks.items() - for qm_name, qm_url in quickmarks: - self.new_item(cat, qm_name, qm_url) + model = base.CompletionModel() + cat = model.new_category("Quickmarks") + quickmarks = objreg.get('quickmark-manager').marks.items() + for qm_name, qm_url in quickmarks: + model.new_item(cat, qm_name, qm_url) + return model -class BookmarkCompletionModel(base.BaseCompletionModel): - +def bookmark(): """A CompletionModel filled with all bookmarks.""" - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - def __init__(self, parent=None): - super().__init__(parent) - cat = self.new_category("Bookmarks") - bookmarks = objreg.get('bookmark-manager').marks.items() - for bm_url, bm_title in bookmarks: - self.new_item(cat, bm_url, bm_title) + model = base.CompletionModel() + cat = model.new_category("Bookmarks") + bookmarks = objreg.get('bookmark-manager').marks.items() + for bm_url, bm_title in bookmarks: + model.new_item(cat, bm_url, bm_title) + return model -class SessionCompletionModel(base.BaseCompletionModel): - +def session(): """A CompletionModel filled with session names.""" - - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method - - def __init__(self, parent=None): - super().__init__(parent) - cat = self.new_category("Sessions") - try: - for name in objreg.get('session-manager').list_sessions(): - if not name.startswith('_'): - self.new_item(cat, name) - except OSError: - log.completion.exception("Failed to list sessions!") + model = base.CompletionModel() + cat = model.new_category("Sessions") + try: + for name in objreg.get('session-manager').list_sessions(): + if not name.startswith('_'): + model.new_item(cat, name) + except OSError: + log.completion.exception("Failed to list sessions!") + return model -class TabCompletionModel(base.BaseCompletionModel): - +def buffer(): """A model to complete on open tabs across all windows. Used for switching the buffer command. """ + idx_column = 0 + url_column = 1 + text_column = 2 - IDX_COLUMN = 0 - URL_COLUMN = 1 - TEXT_COLUMN = 2 - - COLUMN_WIDTHS = (6, 40, 54) - DUMB_SORT = Qt.DescendingOrder - - def __init__(self, parent=None): - super().__init__(parent) - - self.columns_to_filter = [self.IDX_COLUMN, self.URL_COLUMN, - self.TEXT_COLUMN] - - for win_id in objreg.window_registry: - tabbed_browser = objreg.get('tabbed-browser', scope='window', - window=win_id) - for i in range(tabbed_browser.count()): - tab = tabbed_browser.widget(i) - tab.url_changed.connect(self.rebuild) - tab.title_changed.connect(self.rebuild) - tab.shutting_down.connect(self.delayed_rebuild) - tabbed_browser.new_tab.connect(self.on_new_tab) - tabbed_browser.tabBar().tabMoved.connect(self.rebuild) - objreg.get("app").new_window.connect(self.on_new_window) - self.rebuild() - - def on_new_window(self, window): - """Add hooks to new windows.""" - window.tabbed_browser.new_tab.connect(self.on_new_tab) - - @pyqtSlot(browsertab.AbstractTab) - def on_new_tab(self, tab): - """Add hooks to new tabs.""" - tab.url_changed.connect(self.rebuild) - tab.title_changed.connect(self.rebuild) - tab.shutting_down.connect(self.delayed_rebuild) - self.rebuild() - - @pyqtSlot() - def delayed_rebuild(self): - """Fire a rebuild indirectly so widgets get a chance to update.""" - QTimer.singleShot(0, self.rebuild) - - @pyqtSlot() - def rebuild(self): - """Rebuild completion model from current tabs. - - Very lazy method of keeping the model up to date. We could connect to - signals for new tab, tab url/title changed, tab close, tab moved and - make sure we handled background loads too ... but iterating over a - few/few dozen/few hundred tabs doesn't take very long at all. - """ - window_count = 0 - for win_id in objreg.window_registry: - tabbed_browser = objreg.get('tabbed-browser', scope='window', - window=win_id) - if not tabbed_browser.shutting_down: - window_count += 1 - - if window_count < self.rowCount(): - self.removeRows(window_count, self.rowCount() - window_count) - - for i, win_id in enumerate(objreg.window_registry): - tabbed_browser = objreg.get('tabbed-browser', scope='window', - window=win_id) - if tabbed_browser.shutting_down: - continue - if i >= self.rowCount(): - c = self.new_category("{}".format(win_id)) - else: - c = self.item(i, 0) - c.setData("{}".format(win_id), Qt.DisplayRole) - if tabbed_browser.count() < c.rowCount(): - c.removeRows(tabbed_browser.count(), - c.rowCount() - tabbed_browser.count()) - for idx in range(tabbed_browser.count()): - tab = tabbed_browser.widget(idx) - if idx >= c.rowCount(): - self.new_item(c, "{}/{}".format(win_id, idx + 1), - tab.url().toDisplayString(), - tabbed_browser.page_title(idx)) - else: - c.child(idx, 0).setData("{}/{}".format(win_id, idx + 1), - Qt.DisplayRole) - c.child(idx, 1).setData(tab.url().toDisplayString(), - Qt.DisplayRole) - c.child(idx, 2).setData(tabbed_browser.page_title(idx), - Qt.DisplayRole) - - def delete_cur_item(self, completion): - """Delete the selected item. - - Args: - completion: The Completion object to use. - """ + def delete_buffer(completion): + """Close the selected tab.""" index = completion.currentIndex() qtutils.ensure_valid(index) category = index.parent() qtutils.ensure_valid(category) - index = category.child(index.row(), self.IDX_COLUMN) + index = category.child(index.row(), idx_column) win_id, tab_index = index.data().split('/') - tabbed_browser = objreg.get('tabbed-browser', scope='window', window=int(win_id)) tabbed_browser.on_tab_close_requested(int(tab_index) - 1) + model = base.CompletionModel( + column_widths=(6, 40, 54), + dumb_sort=Qt.DescendingOrder, + delete_cur_item=delete_buffer, + columns_to_filter=[idx_column, url_column, text_column]) -class BindCompletionModel(base.BaseCompletionModel): + for win_id in objreg.window_registry: + tabbed_browser = objreg.get('tabbed-browser', scope='window', + window=win_id) + if tabbed_browser.shutting_down: + continue + c = model.new_category("{}".format(win_id)) + for idx in range(tabbed_browser.count()): + tab = tabbed_browser.widget(idx) + model.new_item(c, "{}/{}".format(win_id, idx + 1), + tab.url().toDisplayString(), + tabbed_browser.page_title(idx)) + return model - """A CompletionModel filled with all bindable commands and descriptions.""" - # https://github.com/qutebrowser/qutebrowser/issues/545 - # pylint: disable=abstract-method +def bind(_): + """A CompletionModel filled with all bindable commands and descriptions. - COLUMN_WIDTHS = (20, 60, 20) - - def __init__(self, parent=None): - super().__init__(parent) - cmdlist = _get_cmd_completions(include_hidden=True, - include_aliases=True) - cat = self.new_category("Commands") - for (name, desc, misc) in cmdlist: - self.new_item(cat, name, desc, misc) + Args: + _: the key being bound. + """ + # TODO: offer a 'Current binding' completion based on the key. + model = base.CompletionModel(column_widths=(20, 60, 20)) + cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) + cat = model.new_category("Commands") + for (name, desc, misc) in cmdlist: + model.new_item(cat, name, desc, misc) + return model def _get_cmd_completions(include_hidden, include_aliases, prefix=''): diff --git a/qutebrowser/completion/models/sortfilter.py b/qutebrowser/completion/models/sortfilter.py index e2db88b9e..92dd1b2a0 100644 --- a/qutebrowser/completion/models/sortfilter.py +++ b/qutebrowser/completion/models/sortfilter.py @@ -50,7 +50,7 @@ class CompletionFilterModel(QSortFilterProxyModel): self.pattern = '' self.pattern_re = None - dumb_sort = self.srcmodel.DUMB_SORT + dumb_sort = self.srcmodel.dumb_sort if dumb_sort is None: # pylint: disable=invalid-name self.lessThan = self.intelligentLessThan diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 98f68c08c..d8b7f42a2 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""CompletionModels for URLs.""" +"""Function to return the url completion model for the `open` command.""" import datetime @@ -27,166 +27,172 @@ from qutebrowser.utils import objreg, utils, qtutils, log from qutebrowser.completion.models import base from qutebrowser.config import config +_URL_COLUMN = 0 +_TEXT_COLUMN = 1 +_TIME_COLUMN = 2 +_model = None +_history_cat = None +_quickmark_cat = None +_bookmark_cat = None -class UrlCompletionModel(base.BaseCompletionModel): - """A model which combines bookmarks, quickmarks and web history URLs. +def _delete_url(completion): + index = completion.currentIndex() + qtutils.ensure_valid(index) + category = index.parent() + index = category.child(index.row(), _URL_COLUMN) + qtutils.ensure_valid(category) + if category.data() == 'Bookmarks': + bookmark_manager = objreg.get('bookmark-manager') + bookmark_manager.delete(index.data()) + elif category.data() == 'Quickmarks': + quickmark_manager = objreg.get('quickmark-manager') + sibling = index.sibling(index.row(), _TEXT_COLUMN) + qtutils.ensure_valid(sibling) + name = sibling.data() + quickmark_manager.delete(name) + + +def _remove_oldest_history(): + """Remove the oldest history entry.""" + _history_cat.removeRow(0) + + +def _add_history_entry(entry): + """Add a new history entry to the completion.""" + _model.new_item(_history_cat, entry.url.toDisplayString(), + entry.title, _fmt_atime(entry.atime), + sort=int(entry.atime), userdata=entry.url) + + max_history = config.get('completion', 'web-history-max-items') + if max_history != -1 and _history_cat.rowCount() > max_history: + _remove_oldest_history() + + +@config.change_filter('completion', 'timestamp-format') +def _reformat_timestamps(): + """Reformat the timestamps if the config option was changed.""" + for i in range(_history_cat.rowCount()): + url_item = _history_cat.child(i, _URL_COLUMN) + atime_item = _history_cat.child(i, _TIME_COLUMN) + atime = url_item.data(base.Role.sort) + atime_item.setText(_fmt_atime(atime)) + + +@pyqtSlot(object) +def _on_history_item_added(entry): + """Slot called when a new history item was added.""" + for i in range(_history_cat.rowCount()): + url_item = _history_cat.child(i, _URL_COLUMN) + atime_item = _history_cat.child(i, _TIME_COLUMN) + title_item = _history_cat.child(i, _TEXT_COLUMN) + if url_item.data(base.Role.userdata) == entry.url: + atime_item.setText(_fmt_atime(entry.atime)) + title_item.setText(entry.title) + url_item.setData(int(entry.atime), base.Role.sort) + break + else: + _add_history_entry(entry) + + +@pyqtSlot() +def _on_history_cleared(): + _history_cat.removeRows(0, _history_cat.rowCount()) + + +def _remove_item(data, category, column): + """Helper function for on_quickmark_removed and on_bookmark_removed. + + Args: + data: The item to search for. + category: The category to search in. + column: The column to use for matching. + """ + for i in range(category.rowCount()): + item = category.child(i, column) + if item.data(Qt.DisplayRole) == data: + category.removeRow(i) + break + + +@pyqtSlot(str) +def _on_quickmark_removed(name): + """Called when a quickmark has been removed by the user. + + Args: + name: The name of the quickmark which has been removed. + """ + _remove_item(name, _quickmark_cat, _TEXT_COLUMN) + + +@pyqtSlot(str) +def _on_bookmark_removed(urltext): + """Called when a bookmark has been removed by the user. + + Args: + urltext: The url of the bookmark which has been removed. + """ + _remove_item(urltext, _bookmark_cat, _URL_COLUMN) + + +def _fmt_atime(atime): + """Format an atime to a human-readable string.""" + fmt = config.get('completion', 'timestamp-format') + if fmt is None: + return '' + try: + dt = datetime.datetime.fromtimestamp(atime) + except (ValueError, OSError, OverflowError): + # Different errors which can occur for too large values... + log.misc.error("Got invalid timestamp {}!".format(atime)) + return '(invalid)' + else: + return dt.strftime(fmt) + + +def _init(): + global _model, _quickmark_cat, _bookmark_cat, _history_cat + _model = base.CompletionModel(column_widths=(40, 50, 10), + dumb_sort=Qt.DescendingOrder, + delete_cur_item=_delete_url, + columns_to_filter=[_URL_COLUMN, + _TEXT_COLUMN]) + _quickmark_cat = _model.new_category("Quickmarks") + _bookmark_cat = _model.new_category("Bookmarks") + _history_cat = _model.new_category("History") + + quickmark_manager = objreg.get('quickmark-manager') + quickmarks = quickmark_manager.marks.items() + for qm_name, qm_url in quickmarks: + _model.new_item(_quickmark_cat, qm_url, qm_name) + quickmark_manager.added.connect( + lambda name, url: _model.new_item(_quickmark_cat, url, name)) + quickmark_manager.removed.connect(_on_quickmark_removed) + + bookmark_manager = objreg.get('bookmark-manager') + bookmarks = bookmark_manager.marks.items() + for bm_url, bm_title in bookmarks: + _model.new_item(_bookmark_cat, bm_url, bm_title) + bookmark_manager.added.connect( + lambda name, url: _model.new_item(_bookmark_cat, url, name)) + bookmark_manager.removed.connect(_on_bookmark_removed) + + history = objreg.get('web-history') + max_history = config.get('completion', 'web-history-max-items') + for entry in utils.newest_slice(history, max_history): + if not entry.redirect: + _add_history_entry(entry) + history.add_completion_item.connect(_on_history_item_added) + history.cleared.connect(_on_history_cleared) + + objreg.get('config').changed.connect(_reformat_timestamps) + + +def url(): + """A _model which combines bookmarks, quickmarks and web history URLs. Used for the `open` command. """ - - URL_COLUMN = 0 - TEXT_COLUMN = 1 - TIME_COLUMN = 2 - - COLUMN_WIDTHS = (40, 50, 10) - DUMB_SORT = Qt.DescendingOrder - - def __init__(self, parent=None): - super().__init__(parent) - - self.columns_to_filter = [self.URL_COLUMN, self.TEXT_COLUMN] - - self._quickmark_cat = self.new_category("Quickmarks") - self._bookmark_cat = self.new_category("Bookmarks") - self._history_cat = self.new_category("History") - - quickmark_manager = objreg.get('quickmark-manager') - quickmarks = quickmark_manager.marks.items() - for qm_name, qm_url in quickmarks: - self.new_item(self._quickmark_cat, qm_url, qm_name) - quickmark_manager.added.connect( - lambda name, url: self.new_item(self._quickmark_cat, url, name)) - quickmark_manager.removed.connect(self.on_quickmark_removed) - - bookmark_manager = objreg.get('bookmark-manager') - bookmarks = bookmark_manager.marks.items() - for bm_url, bm_title in bookmarks: - self.new_item(self._bookmark_cat, bm_url, bm_title) - bookmark_manager.added.connect( - lambda name, url: self.new_item(self._bookmark_cat, url, name)) - bookmark_manager.removed.connect(self.on_bookmark_removed) - - self._history = objreg.get('web-history') - self._max_history = config.get('completion', 'web-history-max-items') - history = utils.newest_slice(self._history, self._max_history) - for entry in history: - if not entry.redirect: - self._add_history_entry(entry) - self._history.add_completion_item.connect(self.on_history_item_added) - self._history.cleared.connect(self.on_history_cleared) - - objreg.get('config').changed.connect(self.reformat_timestamps) - - def _fmt_atime(self, atime): - """Format an atime to a human-readable string.""" - fmt = config.get('completion', 'timestamp-format') - if fmt is None: - return '' - try: - dt = datetime.datetime.fromtimestamp(atime) - except (ValueError, OSError, OverflowError): - # Different errors which can occur for too large values... - log.misc.error("Got invalid timestamp {}!".format(atime)) - return '(invalid)' - else: - return dt.strftime(fmt) - - def _remove_oldest_history(self): - """Remove the oldest history entry.""" - self._history_cat.removeRow(0) - - def _add_history_entry(self, entry): - """Add a new history entry to the completion.""" - self.new_item(self._history_cat, entry.url.toDisplayString(), - entry.title, - self._fmt_atime(entry.atime), sort=int(entry.atime), - userdata=entry.url) - - if (self._max_history != -1 and - self._history_cat.rowCount() > self._max_history): - self._remove_oldest_history() - - @config.change_filter('completion', 'timestamp-format') - def reformat_timestamps(self): - """Reformat the timestamps if the config option was changed.""" - for i in range(self._history_cat.rowCount()): - url_item = self._history_cat.child(i, self.URL_COLUMN) - atime_item = self._history_cat.child(i, self.TIME_COLUMN) - atime = url_item.data(base.Role.sort) - atime_item.setText(self._fmt_atime(atime)) - - @pyqtSlot(object) - def on_history_item_added(self, entry): - """Slot called when a new history item was added.""" - for i in range(self._history_cat.rowCount()): - url_item = self._history_cat.child(i, self.URL_COLUMN) - atime_item = self._history_cat.child(i, self.TIME_COLUMN) - title_item = self._history_cat.child(i, self.TEXT_COLUMN) - url = url_item.data(base.Role.userdata) - if url == entry.url: - atime_item.setText(self._fmt_atime(entry.atime)) - title_item.setText(entry.title) - url_item.setData(int(entry.atime), base.Role.sort) - break - else: - self._add_history_entry(entry) - - @pyqtSlot() - def on_history_cleared(self): - self._history_cat.removeRows(0, self._history_cat.rowCount()) - - def _remove_item(self, data, category, column): - """Helper function for on_quickmark_removed and on_bookmark_removed. - - Args: - data: The item to search for. - category: The category to search in. - column: The column to use for matching. - """ - for i in range(category.rowCount()): - item = category.child(i, column) - if item.data(Qt.DisplayRole) == data: - category.removeRow(i) - break - - @pyqtSlot(str) - def on_quickmark_removed(self, name): - """Called when a quickmark has been removed by the user. - - Args: - name: The name of the quickmark which has been removed. - """ - self._remove_item(name, self._quickmark_cat, self.TEXT_COLUMN) - - @pyqtSlot(str) - def on_bookmark_removed(self, url): - """Called when a bookmark has been removed by the user. - - Args: - url: The url of the bookmark which has been removed. - """ - self._remove_item(url, self._bookmark_cat, self.URL_COLUMN) - - def delete_cur_item(self, completion): - """Delete the selected item. - - Args: - completion: The Completion object to use. - """ - index = completion.currentIndex() - qtutils.ensure_valid(index) - category = index.parent() - index = category.child(index.row(), self.URL_COLUMN) - url = index.data() - qtutils.ensure_valid(category) - - if category.data() == 'Bookmarks': - bookmark_manager = objreg.get('bookmark-manager') - bookmark_manager.delete(url) - elif category.data() == 'Quickmarks': - quickmark_manager = objreg.get('quickmark-manager') - sibling = index.sibling(index.row(), self.TEXT_COLUMN) - qtutils.ensure_valid(sibling) - name = sibling.data() - quickmark_manager.delete(name) + if not _model: + _init() + return _model diff --git a/tests/unit/completion/test_column_widths.py b/tests/unit/completion/test_column_widths.py deleted file mode 100644 index 21456ed37..000000000 --- a/tests/unit/completion/test_column_widths.py +++ /dev/null @@ -1,50 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2015-2017 Alexander Cogneau -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Tests for qutebrowser.completion.models column widths.""" - -import pytest - -from qutebrowser.completion.models.base import BaseCompletionModel -from qutebrowser.completion.models.configmodel import ( - SettingOptionCompletionModel, SettingSectionCompletionModel, - SettingValueCompletionModel) -from qutebrowser.completion.models.miscmodels import ( - CommandCompletionModel, HelpCompletionModel, QuickmarkCompletionModel, - BookmarkCompletionModel, SessionCompletionModel) -from qutebrowser.completion.models.urlmodel import UrlCompletionModel - - -CLASSES = [BaseCompletionModel, SettingOptionCompletionModel, - SettingOptionCompletionModel, SettingSectionCompletionModel, - SettingValueCompletionModel, CommandCompletionModel, - HelpCompletionModel, QuickmarkCompletionModel, - BookmarkCompletionModel, SessionCompletionModel, UrlCompletionModel] - - -@pytest.mark.parametrize("model", CLASSES) -def test_list_size(model): - """Test if there are 3 items in the COLUMN_WIDTHS property.""" - assert len(model.COLUMN_WIDTHS) == 3 - - -@pytest.mark.parametrize("model", CLASSES) -def test_column_width_sum(model): - """Test if the sum of the widths asserts to 100.""" - assert sum(model.COLUMN_WIDTHS) == 100 diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index d18e6c125..937107f75 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -34,11 +34,11 @@ class FakeCompletionModel(QStandardItemModel): """Stub for a completion model.""" - DUMB_SORT = None - - def __init__(self, kind, parent=None): + def __init__(self, kind, *pos_args, parent=None): super().__init__(parent) self.kind = kind + self.pos_args = [*pos_args] + self.dumb_sort = None class CompletionWidgetStub(QObject): @@ -74,17 +74,8 @@ def instances(monkeypatch): """Mock the instances module so get returns a fake completion model.""" # populate a model for each completion type, with a nested structure for # option and value completion - instances = {kind: FakeCompletionModel(kind) - for kind in usertypes.Completion} - instances[usertypes.Completion.option] = { - 'general': FakeCompletionModel(usertypes.Completion.option), - } - instances[usertypes.Completion.value] = { - 'general': { - 'editor': FakeCompletionModel(usertypes.Completion.value), - } - } - monkeypatch.setattr(completer, 'instances', instances) + get = lambda kind: lambda *args: FakeCompletionModel(kind, *args) + monkeypatch.setattr(completer, 'instances', get) @pytest.fixture(autouse=True) @@ -140,47 +131,52 @@ def _set_cmd_prompt(cmd, txt): cmd.setCursorPosition(txt.index('|')) -@pytest.mark.parametrize('txt, kind, pattern', [ - (':nope|', usertypes.Completion.command, 'nope'), - (':nope |', None, ''), - (':set |', usertypes.Completion.section, ''), - (':set gen|', usertypes.Completion.section, 'gen'), - (':set general |', usertypes.Completion.option, ''), - (':set what |', None, ''), - (':set general editor |', usertypes.Completion.value, ''), - (':set general editor gv|', usertypes.Completion.value, 'gv'), - (':set general editor "gvim -f"|', usertypes.Completion.value, 'gvim -f'), - (':set general editor "gvim |', usertypes.Completion.value, 'gvim'), - (':set general huh |', None, ''), - (':help |', usertypes.Completion.helptopic, ''), - (':help |', usertypes.Completion.helptopic, ''), - (':open |', usertypes.Completion.url, ''), - (':bind |', None, ''), - (':bind |', usertypes.Completion.command, ''), - (':bind foo|', usertypes.Completion.command, 'foo'), - (':bind | foo', None, ''), - (':set| general ', usertypes.Completion.command, 'set'), - (':|set general ', usertypes.Completion.command, 'set'), - (':set gene|ral ignore-case', usertypes.Completion.section, 'general'), - (':|', usertypes.Completion.command, ''), - (': |', usertypes.Completion.command, ''), - ('/|', None, ''), - (':open -t|', None, ''), - (':open --tab|', None, ''), - (':open -t |', usertypes.Completion.url, ''), - (':open --tab |', usertypes.Completion.url, ''), - (':open | -t', usertypes.Completion.url, ''), - (':tab-detach |', None, ''), - (':bind --mode=caret |', usertypes.Completion.command, ''), +@pytest.mark.parametrize('txt, kind, pattern, pos_args', [ + (':nope|', usertypes.Completion.command, 'nope', []), + (':nope |', None, '', []), + (':set |', usertypes.Completion.section, '', []), + (':set gen|', usertypes.Completion.section, 'gen', []), + (':set general |', usertypes.Completion.option, '', ['general']), + (':set what |', usertypes.Completion.option, '', ['what']), + (':set general editor |', usertypes.Completion.value, '', + ['general', 'editor']), + (':set general editor gv|', usertypes.Completion.value, 'gv', + ['general', 'editor']), + (':set general editor "gvim -f"|', usertypes.Completion.value, 'gvim -f', + ['general', 'editor']), + (':set general editor "gvim |', usertypes.Completion.value, 'gvim', + ['general', 'editor']), + (':set general huh |', usertypes.Completion.value, '', ['general', 'huh']), + (':help |', usertypes.Completion.helptopic, '', []), + (':help |', usertypes.Completion.helptopic, '', []), + (':open |', usertypes.Completion.url, '', []), + (':bind |', None, '', []), + (':bind |', usertypes.Completion.command, '', ['']), + (':bind foo|', usertypes.Completion.command, 'foo', ['']), + (':bind | foo', None, '', []), + (':set| general ', usertypes.Completion.command, 'set', []), + (':|set general ', usertypes.Completion.command, 'set', []), + (':set gene|ral ignore-case', usertypes.Completion.section, 'general', []), + (':|', usertypes.Completion.command, '', []), + (': |', usertypes.Completion.command, '', []), + ('/|', None, '', []), + (':open -t|', None, '', []), + (':open --tab|', None, '', []), + (':open -t |', usertypes.Completion.url, '', []), + (':open --tab |', usertypes.Completion.url, '', []), + (':open | -t', usertypes.Completion.url, '', []), + (':tab-detach |', None, '', []), + (':bind --mode=caret |', usertypes.Completion.command, '', + ['']), pytest.param(':bind --mode caret |', usertypes.Completion.command, - '', marks=pytest.mark.xfail(reason='issue #74')), - (':set -t -p |', usertypes.Completion.section, ''), - (':open -- |', None, ''), - (':gibberish nonesense |', None, ''), - ('/:help|', None, ''), + '', [], marks=pytest.mark.xfail(reason='issue #74')), + (':set -t -p |', usertypes.Completion.section, '', []), + (':open -- |', None, '', []), + (':gibberish nonesense |', None, '', []), + ('/:help|', None, '', []), ('::bind|', usertypes.Completion.command, ':bind'), ]) -def test_update_completion(txt, kind, pattern, status_command_stub, +def test_update_completion(txt, kind, pattern, pos_args, status_command_stub, completer_obj, completion_widget_stub): """Test setting the completion widget's model based on command text.""" # this test uses | as a placeholder for the current cursor position @@ -192,7 +188,9 @@ def test_update_completion(txt, kind, pattern, status_command_stub, if kind is None: assert args[0] is None else: - assert args[0].srcmodel.kind == kind + model = args[0].srcmodel + assert model.kind == kind + assert model.pos_args == pos_args assert args[1] == pattern diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 9a8de3cad..88f4aed20 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -71,7 +71,7 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry, def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" - model = base.BaseCompletionModel() + model = base.CompletionModel() filtermodel = sortfilter.CompletionFilterModel(model) for i in range(3): model.appendRow(QStandardItem(str(i))) @@ -82,7 +82,7 @@ def test_set_model(completionview): def test_set_pattern(completionview): - model = sortfilter.CompletionFilterModel(base.BaseCompletionModel()) + model = sortfilter.CompletionFilterModel(base.CompletionModel()) model.set_pattern = unittest.mock.Mock() completionview.set_model(model, 'foo') model.set_pattern.assert_called_with('foo') @@ -148,7 +148,7 @@ def test_completion_item_focus(which, tree, expected, completionview, qtbot): successive movement. None implies no signal should be emitted. """ - model = base.BaseCompletionModel() + model = base.CompletionModel() for catdata in tree: cat = QStandardItem() model.appendRow(cat) @@ -176,7 +176,7 @@ def test_completion_item_focus_no_model(which, completionview, qtbot): """ with qtbot.assertNotEmitted(completionview.selection_changed): completionview.completion_item_focus(which) - model = base.BaseCompletionModel() + model = base.CompletionModel() filtermodel = sortfilter.CompletionFilterModel(model, parent=completionview) completionview.set_model(filtermodel) @@ -200,7 +200,7 @@ def test_completion_show(show, rows, quick_complete, completionview, config_stub.data['completion']['show'] = show config_stub.data['completion']['quick-complete'] = quick_complete - model = base.BaseCompletionModel() + model = base.CompletionModel() for name in rows: cat = QStandardItem() model.appendRow(cat) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index ff00a11a9..9870bb4db 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -56,6 +56,9 @@ def _check_completions(model, expected): misc = actual_cat.child(j, 2) actual_item = (name.text(), desc.text(), misc.text()) assert actual_item in expected_cat + # sanity-check the column_widths + assert len(model.column_widths) == 3 + assert sum(model.column_widths) == 100 def _patch_cmdutils(monkeypatch, stubs, symbol): @@ -184,7 +187,7 @@ def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, key_config_stub.set_bindings_for('normal', {'s': 'stop', 'rr': 'roll', 'ro': 'rock'}) - model = miscmodels.CommandCompletionModel() + model = miscmodels.command() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -212,7 +215,7 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): key_config_stub.set_bindings_for('normal', {'s': 'stop', 'rr': 'roll'}) _patch_cmdutils(monkeypatch, stubs, module + '.cmdutils') _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') - model = miscmodels.HelpCompletionModel() + model = miscmodels.helptopic() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -236,7 +239,7 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" - model = miscmodels.QuickmarkCompletionModel() + model = miscmodels.quickmark() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -251,7 +254,7 @@ def test_quickmark_completion(qtmodeltester, quickmarks): def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" - model = miscmodels.BookmarkCompletionModel() + model = miscmodels.bookmark() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -275,7 +278,7 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, """ config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 2} - model = urlmodel.UrlCompletionModel() + model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -303,7 +306,7 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 2} - model = urlmodel.UrlCompletionModel() + model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -321,7 +324,7 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 2} - model = urlmodel.UrlCompletionModel() + model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -335,7 +338,7 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, def test_session_completion(qtmodeltester, session_manager_stub): session_manager_stub.sessions = ['default', '1', '2'] - model = miscmodels.SessionCompletionModel() + model = miscmodels.session() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -354,7 +357,7 @@ def test_tab_completion(qtmodeltester, fake_web_tab, app_stub, win_registry, tabbed_browser_stubs[1].tabs = [ fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0), ] - model = miscmodels.TabCompletionModel() + model = miscmodels.buffer() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -381,7 +384,7 @@ def test_tab_completion_delete(qtmodeltester, fake_web_tab, qtbot, app_stub, tabbed_browser_stubs[1].tabs = [ fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0), ] - model = miscmodels.TabCompletionModel() + model = miscmodels.buffer() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -398,7 +401,7 @@ def test_setting_section_completion(qtmodeltester, monkeypatch, stubs): _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') _patch_config_section_desc(monkeypatch, stubs, module + '.configdata.SECTION_DESC') - model = configmodel.SettingSectionCompletionModel() + model = configmodel.section() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -418,7 +421,7 @@ def test_setting_option_completion(qtmodeltester, monkeypatch, stubs, config_stub.data = {'ui': {'gesture': 'off', 'mind': 'on', 'voice': 'sometimes'}} - model = configmodel.SettingOptionCompletionModel('ui') + model = configmodel.option('ui') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -440,7 +443,7 @@ def test_setting_option_completion_valuelist(qtmodeltester, monkeypatch, stubs, 'DEFAULT': 'https://duckduckgo.com/?q={}' } } - model = configmodel.SettingOptionCompletionModel('searchengines') + model = configmodel.option('searchengines') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -454,7 +457,7 @@ def test_setting_value_completion(qtmodeltester, monkeypatch, stubs, module = 'qutebrowser.completion.models.configmodel' _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') config_stub.data = {'general': {'volume': '0'}} - model = configmodel.SettingValueCompletionModel('general', 'volume') + model = configmodel.value('general', 'volume') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -486,7 +489,7 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, key_config_stub.set_bindings_for('normal', {'s': 'stop', 'rr': 'roll', 'ro': 'rock'}) - model = miscmodels.BindCompletionModel() + model = miscmodels.bind('s') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) diff --git a/tests/unit/completion/test_sortfilter.py b/tests/unit/completion/test_sortfilter.py index 2d4a4e25d..c972a5ec9 100644 --- a/tests/unit/completion/test_sortfilter.py +++ b/tests/unit/completion/test_sortfilter.py @@ -33,7 +33,7 @@ def _create_model(data): tuple in the sub-list represents an item, and each value in the tuple represents the item data for that column """ - model = base.BaseCompletionModel() + model = base.CompletionModel() for catdata in data: cat = model.new_category('') for itemdata in catdata: @@ -74,7 +74,7 @@ def _extract_model_data(model): ('4', 'blah', False), ]) def test_filter_accepts_row(pattern, data, expected): - source_model = base.BaseCompletionModel() + source_model = base.CompletionModel() cat = source_model.new_category('test') source_model.new_item(cat, data) @@ -119,8 +119,8 @@ def test_first_last_item(tree, first, last): def test_set_source_model(): """Ensure setSourceModel sets source_model and clears the pattern.""" - model1 = base.BaseCompletionModel() - model2 = base.BaseCompletionModel() + model1 = base.CompletionModel() + model2 = base.CompletionModel() filter_model = sortfilter.CompletionFilterModel(model1) filter_model.set_pattern('foo') # sourceModel() is cached as srcmodel, so make sure both match @@ -202,7 +202,7 @@ def test_count(tree, expected): def test_set_pattern(pattern, dumb_sort, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" model = _create_model(before) - model.DUMB_SORT = dumb_sort + model.dumb_sort = dumb_sort model.columns_to_filter = filter_cols filter_model = sortfilter.CompletionFilterModel(model) filter_model.set_pattern(pattern) From b36cf0572dca4a0cd64a8a1f5539ddc61dc18372 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 19 Sep 2016 12:44:51 -0400 Subject: [PATCH 002/337] Avoid potential circular import in config.py. There was a circular import from config -> keyconf -> miscmodels -> config. This is resolved by scoping config's keyconf import to the one function that uses it. --- qutebrowser/config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index 587da214f..bb52a2417 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -38,7 +38,6 @@ from PyQt5.QtCore import pyqtSignal, QObject, QUrl, QSettings from PyQt5.QtGui import QColor from qutebrowser.config import configdata, configexc, textwrapper -from qutebrowser.config.parsers import keyconf from qutebrowser.config.parsers import ini from qutebrowser.commands import cmdexc, cmdutils from qutebrowser.utils import (message, objreg, utils, standarddir, log, @@ -181,6 +180,7 @@ def _init_key_config(parent): Args: parent: The parent to use for the KeyConfigParser. """ + from qutebrowser.config.parsers import keyconf args = objreg.get('args') try: key_config = keyconf.KeyConfigParser(standarddir.config(), 'keys.conf', From 3b30b422118c918fc19fcbc4837be28b00387fa8 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Sep 2016 06:52:52 -0400 Subject: [PATCH 003/337] Remove completion.instances, usertypes.Completion. The new completion API no longer needs either of these. Instead of referencing an enum member, cmdutils.argument.completion now points to a function that returnsthe desired completion model. This vastly simplifies the addition of new completion types. Previously it was necessary to define the new model as well as editing usertypes and completion.models.instances. Now it is only necessary to define a single function under completion.models. This is the next step of Completion Model/View Revamping (#74). --- qutebrowser/app.py | 7 +- qutebrowser/browser/commands.py | 20 ++-- qutebrowser/completion/completer.py | 8 +- qutebrowser/completion/models/instances.py | 53 ----------- qutebrowser/config/config.py | 8 +- qutebrowser/config/parsers/keyconf.py | 5 +- qutebrowser/misc/sessions.py | 11 ++- qutebrowser/utils/usertypes.py | 7 -- tests/unit/completion/test_completer.py | 102 +++++++++++---------- 9 files changed, 84 insertions(+), 137 deletions(-) delete mode 100644 qutebrowser/completion/models/instances.py diff --git a/qutebrowser/app.py b/qutebrowser/app.py index a3a855f9a..2b216ca44 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -42,7 +42,7 @@ except ImportError: import qutebrowser import qutebrowser.resources -from qutebrowser.completion.models import instances as completionmodels +from qutebrowser.completion.models import miscmodels from qutebrowser.commands import cmdutils, runners, cmdexc from qutebrowser.config import style, config, websettings, configexc from qutebrowser.browser import (urlmarks, adblock, history, browsertab, @@ -459,9 +459,6 @@ def _init_modules(args, crash_handler): diskcache = cache.DiskCache(standarddir.cache(), parent=qApp) objreg.register('cache', diskcache) - log.init.debug("Initializing completions...") - completionmodels.init() - log.init.debug("Misc initialization...") if config.get('ui', 'hide-wayland-decoration'): os.environ['QT_WAYLAND_DISABLE_WINDOWDECORATION'] = '1' @@ -753,7 +750,7 @@ class Quitter: QTimer.singleShot(0, functools.partial(qApp.exit, status)) @cmdutils.register(instance='quitter', name='wq') - @cmdutils.argument('name', completion=usertypes.Completion.sessions) + @cmdutils.argument('name', completion=miscmodels.session) def save_and_quit(self, name=sessions.default): """Save open pages and quit. diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 97a764c68..8914d4ac2 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -41,7 +41,7 @@ from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils, objreg, utils, typing) from qutebrowser.utils.usertypes import KeyMode from qutebrowser.misc import editor, guiprocess -from qutebrowser.completion.models import instances, sortfilter +from qutebrowser.completion.models import sortfilter, urlmodel, miscmodels class CommandDispatcher: @@ -284,7 +284,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', name='open', maxsplit=0, scope='window') - @cmdutils.argument('url', completion=usertypes.Completion.url) + @cmdutils.argument('url', completion=urlmodel.url) @cmdutils.argument('count', count=True) def openurl(self, url=None, implicit=False, bg=False, tab=False, window=False, count=None, secure=False, @@ -1007,7 +1007,7 @@ class CommandDispatcher: self._open(url, tab, bg, window) @cmdutils.register(instance='command-dispatcher', scope='window') - @cmdutils.argument('index', completion=usertypes.Completion.tab) + @cmdutils.argument('index', completion=miscmodels.buffer) def buffer(self, index): """Select tab by index or url/title best match. @@ -1023,7 +1023,7 @@ class CommandDispatcher: for part in index_parts: int(part) except ValueError: - model = instances.get(usertypes.Completion.tab) + model = miscmodels.buffer() sf = sortfilter.CompletionFilterModel(source=model) sf.set_pattern(index) if sf.count() > 0: @@ -1229,8 +1229,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0) - @cmdutils.argument('name', - completion=usertypes.Completion.quickmark_by_name) + @cmdutils.argument('name', completion=miscmodels.quickmark) def quickmark_load(self, name, tab=False, bg=False, window=False): """Load a quickmark. @@ -1248,8 +1247,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0) - @cmdutils.argument('name', - completion=usertypes.Completion.quickmark_by_name) + @cmdutils.argument('name', completion=miscmodels.quickmark) def quickmark_del(self, name=None): """Delete a quickmark. @@ -1311,7 +1309,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0) - @cmdutils.argument('url', completion=usertypes.Completion.bookmark_by_url) + @cmdutils.argument('url', completion=miscmodels.bookmark) def bookmark_load(self, url, tab=False, bg=False, window=False, delete=False): """Load a bookmark. @@ -1333,7 +1331,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0) - @cmdutils.argument('url', completion=usertypes.Completion.bookmark_by_url) + @cmdutils.argument('url', completion=miscmodels.bookmark) def bookmark_del(self, url=None): """Delete a bookmark. @@ -1507,7 +1505,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', name='help', scope='window') - @cmdutils.argument('topic', completion=usertypes.Completion.helptopic) + @cmdutils.argument('topic', completion=miscmodels.helptopic) def show_help(self, tab=False, bg=False, window=False, topic=None): r"""Show help about a command or setting. diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 21af74da5..bd4f43009 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -23,8 +23,8 @@ from PyQt5.QtCore import pyqtSlot, QObject, QTimer from qutebrowser.config import config from qutebrowser.commands import cmdutils, runners -from qutebrowser.utils import usertypes, log, utils -from qutebrowser.completion.models import instances, sortfilter +from qutebrowser.utils import log, utils +from qutebrowser.completion.models import sortfilter, miscmodels class Completer(QObject): @@ -72,7 +72,7 @@ class Completer(QObject): Return: A completion model or None. """ - model = instances.get(completion)(*pos_args) + model = completion(*pos_args) if model is None: return None else: @@ -96,7 +96,7 @@ class Completer(QObject): log.completion.debug("After removing flags: {}".format(before_cursor)) if not before_cursor: # '|' or 'set|' - model = instances.get(usertypes.Completion.command)() + model = miscmodels.command() return sortfilter.CompletionFilterModel(source=model, parent=self) try: cmd = cmdutils.cmd_dict[before_cursor[0]] diff --git a/qutebrowser/completion/models/instances.py b/qutebrowser/completion/models/instances.py deleted file mode 100644 index ce109ae7a..000000000 --- a/qutebrowser/completion/models/instances.py +++ /dev/null @@ -1,53 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2015-2017 Florian Bruhin (The Compiler) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Global instances of the completion models.""" - -from qutebrowser.utils import usertypes -from qutebrowser.completion.models import miscmodels, urlmodel, configmodel - - -def get(completion): - """Get a certain completion. Initializes the completion if needed.""" - if completion == usertypes.Completion.command: - return miscmodels.command - if completion == usertypes.Completion.helptopic: - return miscmodels.helptopic - if completion == usertypes.Completion.tab: - return miscmodels.buffer - if completion == usertypes.Completion.quickmark_by_name: - return miscmodels.quickmark - if completion == usertypes.Completion.bookmark_by_url: - return miscmodels.bookmark - if completion == usertypes.Completion.sessions: - return miscmodels.session - if completion == usertypes.Completion.bind: - return miscmodels.bind - if completion == usertypes.Completion.section: - return configmodel.section - if completion == usertypes.Completion.option: - return configmodel.option - if completion == usertypes.Completion.value: - return configmodel.value - if completion == usertypes.Completion.url: - return urlmodel.url - - -def init(): - pass diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index bb52a2417..dd71f5333 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -43,7 +43,7 @@ from qutebrowser.commands import cmdexc, cmdutils from qutebrowser.utils import (message, objreg, utils, standarddir, log, qtutils, error, usertypes) from qutebrowser.misc import objects -from qutebrowser.utils.usertypes import Completion +from qutebrowser.completion.models import configmodel UNSET = object() @@ -794,9 +794,9 @@ class ConfigManager(QObject): e.__class__.__name__, e)) @cmdutils.register(name='set', instance='config', star_args_optional=True) - @cmdutils.argument('section_', completion=Completion.section) - @cmdutils.argument('option', completion=Completion.option) - @cmdutils.argument('values', completion=Completion.value) + @cmdutils.argument('section_', completion=configmodel.section) + @cmdutils.argument('option', completion=configmodel.option) + @cmdutils.argument('values', completion=configmodel.value) @cmdutils.argument('win_id', win_id=True) def set_command(self, win_id, section_=None, option=None, *values, temp=False, print_=False): diff --git a/qutebrowser/config/parsers/keyconf.py b/qutebrowser/config/parsers/keyconf.py index 53f23d7c0..e4adb8676 100644 --- a/qutebrowser/config/parsers/keyconf.py +++ b/qutebrowser/config/parsers/keyconf.py @@ -27,7 +27,8 @@ from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.config import configdata, textwrapper from qutebrowser.commands import cmdutils, cmdexc -from qutebrowser.utils import log, utils, qtutils, message, usertypes +from qutebrowser.utils import log, utils, qtutils, message +from qutebrowser.completion.models import miscmodels class KeyConfigError(Exception): @@ -153,7 +154,7 @@ class KeyConfigParser(QObject): @cmdutils.register(instance='key-config', maxsplit=1, no_cmd_split=True, no_replace_variables=True) - @cmdutils.argument('command', completion=usertypes.Completion.bind) + @cmdutils.argument('command', completion=miscmodels.bind) def bind(self, key, command=None, *, mode='normal', force=False): """Bind a key to a command. diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index aa9a1be97..124b78053 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -31,10 +31,11 @@ try: except ImportError: # pragma: no cover from yaml import SafeLoader as YamlLoader, SafeDumper as YamlDumper -from qutebrowser.utils import (standarddir, objreg, qtutils, log, usertypes, - message, utils) +from qutebrowser.utils import (standarddir, objreg, qtutils, log, message, + utils) from qutebrowser.commands import cmdexc, cmdutils from qutebrowser.config import config +from qutebrowser.completion.models import miscmodels default = object() # Sentinel value @@ -433,7 +434,7 @@ class SessionManager(QObject): return sessions @cmdutils.register(instance='session-manager') - @cmdutils.argument('name', completion=usertypes.Completion.sessions) + @cmdutils.argument('name', completion=miscmodels.session) def session_load(self, name, clear=False, temp=False, force=False): """Load a session. @@ -461,7 +462,7 @@ class SessionManager(QObject): win.close() @cmdutils.register(name=['session-save', 'w'], instance='session-manager') - @cmdutils.argument('name', completion=usertypes.Completion.sessions) + @cmdutils.argument('name', completion=miscmodels.session) @cmdutils.argument('win_id', win_id=True) @cmdutils.argument('with_private', flag='p') def session_save(self, name: str = default, current=False, quiet=False, @@ -500,7 +501,7 @@ class SessionManager(QObject): message.info("Saved session {}.".format(name)) @cmdutils.register(instance='session-manager') - @cmdutils.argument('name', completion=usertypes.Completion.sessions) + @cmdutils.argument('name', completion=miscmodels.session) def session_delete(self, name, force=False): """Delete a session. diff --git a/qutebrowser/utils/usertypes.py b/qutebrowser/utils/usertypes.py index 7d31ba6ac..31f2f79cb 100644 --- a/qutebrowser/utils/usertypes.py +++ b/qutebrowser/utils/usertypes.py @@ -236,13 +236,6 @@ KeyMode = enum('KeyMode', ['normal', 'hint', 'command', 'yesno', 'prompt', 'jump_mark', 'record_macro', 'run_macro']) -# Available command completions -Completion = enum('Completion', ['command', 'section', 'option', 'value', - 'helptopic', 'quickmark_by_name', - 'bookmark_by_url', 'url', 'tab', 'sessions', - 'bind']) - - # Exit statuses for errors. Needs to be an int for sys.exit. Exit = enum('Exit', ['ok', 'reserved', 'exception', 'err_ipc', 'err_init', 'err_config', 'err_key_config'], is_int=True, start=0) diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index 937107f75..48f619104 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -26,7 +26,6 @@ from PyQt5.QtCore import QObject from PyQt5.QtGui import QStandardItemModel from qutebrowser.completion import completer -from qutebrowser.utils import usertypes from qutebrowser.commands import command, cmdutils @@ -70,30 +69,45 @@ def completer_obj(qtbot, status_command_stub, config_stub, monkeypatch, stubs, @pytest.fixture(autouse=True) -def instances(monkeypatch): - """Mock the instances module so get returns a fake completion model.""" - # populate a model for each completion type, with a nested structure for - # option and value completion - get = lambda kind: lambda *args: FakeCompletionModel(kind, *args) - monkeypatch.setattr(completer, 'instances', get) +def miscmodels_patch(mocker): + """Patch the miscmodels module to provide fake completion functions. + + Technically some of these are not part of miscmodels, but rolling them into + one module is easier and sufficient for mocking. The only one referenced + directly by Completer is miscmodels.command. + """ + m = mocker.patch('qutebrowser.completion.completer.miscmodels', + autospec=True) + m.command = lambda *args: FakeCompletionModel('command', *args) + m.helptopic = lambda *args: FakeCompletionModel('helptopic', *args) + m.quickmark = lambda *args: FakeCompletionModel('quickmark', *args) + m.bookmark = lambda *args: FakeCompletionModel('bookmark', *args) + m.session = lambda *args: FakeCompletionModel('session', *args) + m.buffer = lambda *args: FakeCompletionModel('buffer', *args) + m.bind = lambda *args: FakeCompletionModel('bind', *args) + m.url = lambda *args: FakeCompletionModel('url', *args) + m.section = lambda *args: FakeCompletionModel('section', *args) + m.option = lambda *args: FakeCompletionModel('option', *args) + m.value = lambda *args: FakeCompletionModel('value', *args) + return m @pytest.fixture(autouse=True) -def cmdutils_patch(monkeypatch, stubs): +def cmdutils_patch(monkeypatch, stubs, miscmodels_patch): """Patch the cmdutils module to provide fake commands.""" - @cmdutils.argument('section_', completion=usertypes.Completion.section) - @cmdutils.argument('option', completion=usertypes.Completion.option) - @cmdutils.argument('value', completion=usertypes.Completion.value) + @cmdutils.argument('section_', completion=miscmodels_patch.section) + @cmdutils.argument('option', completion=miscmodels_patch.option) + @cmdutils.argument('value', completion=miscmodels_patch.value) def set_command(section_=None, option=None, value=None): """docstring.""" pass - @cmdutils.argument('topic', completion=usertypes.Completion.helptopic) + @cmdutils.argument('topic', completion=miscmodels_patch.helptopic) def show_help(tab=False, bg=False, window=False, topic=None): """docstring.""" pass - @cmdutils.argument('url', completion=usertypes.Completion.url) + @cmdutils.argument('url', completion=miscmodels_patch.url) @cmdutils.argument('count', count=True) def openurl(url=None, implicit=False, bg=False, tab=False, window=False, count=None): @@ -101,7 +115,7 @@ def cmdutils_patch(monkeypatch, stubs): pass @cmdutils.argument('win_id', win_id=True) - @cmdutils.argument('command', completion=usertypes.Completion.command) + @cmdutils.argument('command', completion=miscmodels_patch.command) def bind(key, win_id, command=None, *, mode='normal', force=False): """docstring.""" pass @@ -132,49 +146,45 @@ def _set_cmd_prompt(cmd, txt): @pytest.mark.parametrize('txt, kind, pattern, pos_args', [ - (':nope|', usertypes.Completion.command, 'nope', []), + (':nope|', 'command', 'nope', []), (':nope |', None, '', []), - (':set |', usertypes.Completion.section, '', []), - (':set gen|', usertypes.Completion.section, 'gen', []), - (':set general |', usertypes.Completion.option, '', ['general']), - (':set what |', usertypes.Completion.option, '', ['what']), - (':set general editor |', usertypes.Completion.value, '', + (':set |', 'section', '', []), + (':set gen|', 'section', 'gen', []), + (':set general |', 'option', '', ['general']), + (':set what |', 'option', '', ['what']), + (':set general editor |', 'value', '', ['general', 'editor']), + (':set general editor gv|', 'value', 'gv', ['general', 'editor']), + (':set general editor "gvim -f"|', 'value', 'gvim -f', ['general', 'editor']), - (':set general editor gv|', usertypes.Completion.value, 'gv', - ['general', 'editor']), - (':set general editor "gvim -f"|', usertypes.Completion.value, 'gvim -f', - ['general', 'editor']), - (':set general editor "gvim |', usertypes.Completion.value, 'gvim', - ['general', 'editor']), - (':set general huh |', usertypes.Completion.value, '', ['general', 'huh']), - (':help |', usertypes.Completion.helptopic, '', []), - (':help |', usertypes.Completion.helptopic, '', []), - (':open |', usertypes.Completion.url, '', []), + (':set general editor "gvim |', 'value', 'gvim', ['general', 'editor']), + (':set general huh |', 'value', '', ['general', 'huh']), + (':help |', 'helptopic', '', []), + (':help |', 'helptopic', '', []), + (':open |', 'url', '', []), (':bind |', None, '', []), - (':bind |', usertypes.Completion.command, '', ['']), - (':bind foo|', usertypes.Completion.command, 'foo', ['']), + (':bind |', 'command', '', ['']), + (':bind foo|', 'command', 'foo', ['']), (':bind | foo', None, '', []), - (':set| general ', usertypes.Completion.command, 'set', []), - (':|set general ', usertypes.Completion.command, 'set', []), - (':set gene|ral ignore-case', usertypes.Completion.section, 'general', []), - (':|', usertypes.Completion.command, '', []), - (': |', usertypes.Completion.command, '', []), + (':set| general ', 'command', 'set', []), + (':|set general ', 'command', 'set', []), + (':set gene|ral ignore-case', 'section', 'general', []), + (':|', 'command', '', []), + (': |', 'command', '', []), ('/|', None, '', []), (':open -t|', None, '', []), (':open --tab|', None, '', []), - (':open -t |', usertypes.Completion.url, '', []), - (':open --tab |', usertypes.Completion.url, '', []), - (':open | -t', usertypes.Completion.url, '', []), + (':open -t |', 'url', '', []), + (':open --tab |', 'url', '', []), + (':open | -t', 'url', '', []), (':tab-detach |', None, '', []), - (':bind --mode=caret |', usertypes.Completion.command, '', - ['']), - pytest.param(':bind --mode caret |', usertypes.Completion.command, - '', [], marks=pytest.mark.xfail(reason='issue #74')), - (':set -t -p |', usertypes.Completion.section, '', []), + (':bind --mode=caret |', 'command', '', ['']), + pytest.param(':bind --mode caret |', 'command', '', [], + marks=pytest.mark.xfail(reason='issue #74')), + (':set -t -p |', 'section', '', []), (':open -- |', None, '', []), (':gibberish nonesense |', None, '', []), ('/:help|', None, '', []), - ('::bind|', usertypes.Completion.command, ':bind'), + ('::bind|', 'command', ':bind', []), ]) def test_update_completion(txt, kind, pattern, pos_args, status_command_stub, completer_obj, completion_widget_stub): From f43f78c40fd61a4cc24e088b94483ae21f43c206 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 23 Dec 2016 12:47:36 -0500 Subject: [PATCH 004/337] Implement SQL interface. When qutebrowser starts, it creates an in-memory sqlite database. One can instantiate a SqlTable to create a new table in the database. The object provides an interface to query and modify the table. This intended to serve as the base class for the quickmark, bookmark, and history manager objects in objreg. Instead of reading their data into an in-memory dict, they will read into an in-memory sql table. Eventually the completion models for history, bookmarks, and quickmarks can be replaced with SqlQuery models for faster creation and filtering. See #1765. --- qutebrowser/app.py | 5 +- qutebrowser/misc/sql.py | 166 ++++++++++++++++++++++++++++++++++++ tests/unit/misc/test_sql.py | 126 +++++++++++++++++++++++++++ 3 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 qutebrowser/misc/sql.py create mode 100644 tests/unit/misc/test_sql.py diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 2b216ca44..8b2aa7d69 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -53,7 +53,7 @@ from qutebrowser.browser.webkit.network import networkmanager from qutebrowser.keyinput import macros from qutebrowser.mainwindow import mainwindow, prompt from qutebrowser.misc import (readline, ipc, savemanager, sessions, - crashsignal, earlyinit, objects) + crashsignal, earlyinit, objects, sql) from qutebrowser.misc import utilcmds # pylint: disable=unused-import from qutebrowser.utils import (log, version, message, utils, qtutils, urlutils, objreg, usertypes, standarddir, error, debug) @@ -423,6 +423,9 @@ def _init_modules(args, crash_handler): config.init(qApp) save_manager.init_autosave() + log.init.debug("Initializing sql...") + sql.init() + log.init.debug("Initializing web history...") history.init(qApp) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py new file mode 100644 index 000000000..fd7257bdb --- /dev/null +++ b/qutebrowser/misc/sql.py @@ -0,0 +1,166 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Provides access to an in-memory sqlite database.""" + +from PyQt5.QtCore import QObject, pyqtSignal +from PyQt5.QtSql import QSqlDatabase, QSqlQuery + +from qutebrowser.utils import log + + +def init(): + """Initialize the SQL database connection.""" + database = QSqlDatabase.addDatabase('QSQLITE') + # In-memory database, see https://sqlite.org/inmemorydb.html + database.setDatabaseName(':memory:') + if not database.open(): + raise SqlException("Failed to open in-memory sqlite database") + + +def close(): + """Close the SQL connection.""" + QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) + + +def _run_query(querystr, *values): + """Run the given SQL query string on the database. + + Args: + values: positional parameter bindings. + """ + log.completion.debug('Running SQL query: "{}"'.format(querystr)) + database = QSqlDatabase.database() + query = QSqlQuery(database) + query.prepare(querystr) + for val in values: + query.addBindValue(val) + log.completion.debug('Query bindings: {}'.format(query.boundValues())) + if not query.exec_(): + raise SqlException('Failed to exec query "{}": "{}"'.format( + querystr, query.lastError().text())) + return query + + +class SqlTable(QObject): + + """Interface to a sql table. + + Attributes: + _name: Name of the SQL table this wraps. + _primary_key: The primary key of the table. + + Signals: + changed: Emitted when the table is modified. + """ + + changed = pyqtSignal() + + def __init__(self, name, fields, primary_key, parent=None): + """Create a new table in the sql database. + + Raises SqlException if the table already exists. + + Args: + name: Name of the table. + fields: A list of field names. + primary_key: Name of the field to serve as the primary key. + """ + super().__init__(parent) + self._name = name + self._primary_key = primary_key + _run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))" + .format(name, ','.join(fields), primary_key)) + + def __iter__(self): + """Iterate rows in the table.""" + result = _run_query("SELECT * FROM {}".format(self._name)) + while result.next(): + rec = result.record() + yield tuple(rec.value(i) for i in range(rec.count())) + + def __contains__(self, key): + """Return whether the table contains the matching item. + + Args: + key: Primary key value to search for. + """ + query = _run_query("SELECT * FROM {} where {} = ?" + .format(self._name, self._primary_key), key) + return query.next() + + def __len__(self): + """Return the count of rows in the table.""" + result = _run_query("SELECT count(*) FROM {}".format(self._name)) + result.next() + return result.value(0) + + def __getitem__(self, key): + """Retrieve the row matching the given key. + + Args: + key: Primary key value to fetch. + """ + result = _run_query("SELECT * FROM {} where {} = ?" + .format(self._name, self._primary_key), key) + result.next() + rec = result.record() + return tuple(rec.value(i) for i in range(rec.count())) + + def delete(self, value, field=None): + """Remove all rows for which `field` equals `value`. + + Args: + value: Key value to delete. + field: Field to use as the key, defaults to the primary key. + + Return: + The number of rows deleted. + """ + field = field or self._primary_key + query = _run_query("DELETE FROM {} where {} = ?" + .format(self._name, field), value) + if not query.numRowsAffected(): + raise KeyError('No row with {} = "{}"'.format(field, value)) + self.changed.emit() + + def insert(self, *values, replace=False): + """Append a row to the table. + + Args: + values: Values in the order fields were given on table creation. + replace: If true, allow inserting over an existing primary key. + """ + cmd = "REPLACE" if replace else "INSERT" + paramstr = ','.join(['?'] * len(values)) + _run_query("{} INTO {} values({})" + .format(cmd, self._name, paramstr), *values) + self.changed.emit() + + def delete_all(self): + """Remove all row from the table.""" + _run_query("DELETE FROM {}".format(self._name)) + self.changed.emit() + + +class SqlException(Exception): + + """Raised on an error interacting with the SQL database.""" + + pass diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py new file mode 100644 index 000000000..175f2f97e --- /dev/null +++ b/tests/unit/misc/test_sql.py @@ -0,0 +1,126 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Test the SQL API.""" + +import pytest +from qutebrowser.misc import sql + + +@pytest.fixture(autouse=True) +def init(): + sql.init() + yield + sql.close() + + +def test_init(): + sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + with pytest.raises(sql.SqlException): + # table name collision on 'Foo' + sql.SqlTable('Foo', ['foo', 'bar'], primary_key='foo') + + +def test_insert(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + with qtbot.waitSignal(table.changed): + table.insert('one', 1, False) + with qtbot.waitSignal(table.changed): + table.insert('wan', 1, False) + with pytest.raises(sql.SqlException): + # duplicate primary key + table.insert('one', 1, False) + + +def test_iter(): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + table.insert('nine', 9, False) + table.insert('thirteen', 13, True) + assert list(table) == [('one', 1, False), + ('nine', 9, False), + ('thirteen', 13, True)] + + +def test_replace(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + with qtbot.waitSignal(table.changed): + table.insert('one', 1, True, replace=True) + assert list(table) == [('one', 1, True)] + + +def test_delete(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + table.insert('nine', 9, False) + table.insert('thirteen', 13, True) + with pytest.raises(KeyError): + table.delete('nope') + with qtbot.waitSignal(table.changed): + table.delete('thirteen') + assert list(table) == [('one', 1, False), ('nine', 9, False)] + with qtbot.waitSignal(table.changed): + table.delete(False, field='lucky') + assert not list(table) == [('thirteen', 13, True)] + + +def test_len(): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + assert len(table) == 0 + table.insert('one', 1, False) + assert len(table) == 1 + table.insert('nine', 9, False) + assert len(table) == 2 + table.insert('thirteen', 13, True) + assert len(table) == 3 + + +def test_contains(): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + table.insert('nine', 9, False) + table.insert('thirteen', 13, True) + assert 'oone' not in table + assert 'ninee' not in table + assert 1 not in table + assert '*' not in table + assert 'one' in table + assert 'nine' in table + assert 'thirteen' in table + + +def test_index(): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + table.insert('nine', 9, False) + table.insert('thirteen', 13, True) + assert table['one'] == ('one', 1, False) + assert table['nine'] == ('nine', 9, False) + assert table['thirteen'] == ('thirteen', 13, True) + + +def test_delete_all(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table.insert('one', 1, False) + table.insert('nine', 9, False) + table.insert('thirteen', 13, True) + with qtbot.waitSignal(table.changed): + table.delete_all() + assert list(table) == [] From 9477a2eeb2e779d136d9d2d212c2db8a15d905ea Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 15 Jan 2017 21:42:35 -0500 Subject: [PATCH 005/337] Use SQL for history storage. The browser-wide in-memory web history is now stored in an in-memory sql database instead of a python dict. Long-term storage is not affected, it is still persisted in a text file of the same format. This will set the stage for SQL-based history completion. See #1765. --- qutebrowser/browser/history.py | 61 +++-------- qutebrowser/browser/webkit/webkithistory.py | 2 +- tests/unit/browser/webkit/test_history.py | 109 +++++++++----------- 3 files changed, 65 insertions(+), 107 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index aaad08fb3..a1fc7dfa2 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -27,7 +27,7 @@ from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject from qutebrowser.commands import cmdutils from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils, usertypes, message) -from qutebrowser.misc import lineparser, objects +from qutebrowser.misc import lineparser, objects, sql class Entry: @@ -103,19 +103,16 @@ class Entry: return cls(atime, url, title, redirect=redirect) -class WebHistory(QObject): +class WebHistory(sql.SqlTable): """The global history of visited pages. This is a little more complex as you'd expect so the history can be read from disk async while new history is already arriving. - self.history_dict is the main place where the history is stored, in an - OrderedDict (sorted by time) of URL strings mapped to Entry objects. - While reading from disk is still ongoing, the history is saved in - self._temp_history instead, and then appended to self.history_dict once - that's fully populated. + self._temp_history instead, and then inserted into the sql table once + the async read completes. All history which is new in this session (rather than read from disk from a previous browsing session) is also stored in self._new_history. @@ -123,52 +120,34 @@ class WebHistory(QObject): disk, so we can always append to the existing data. Attributes: - history_dict: An OrderedDict of URLs read from the on-disk history. _lineparser: The AppendLineParser used to save the history. _new_history: A list of Entry items of the current session. _saved_count: How many HistoryEntries have been written to disk. _initial_read_started: Whether async_read was called. _initial_read_done: Whether async_read has completed. - _temp_history: OrderedDict of temporary history entries before - async_read was called. + _temp_history: List of history entries from before async_read finished. Signals: - add_completion_item: Emitted before a new Entry is added. - Used to sync with the completion. - arg: The new Entry. - item_added: Emitted after a new Entry is added. - Used to tell the savemanager that the history is dirty. - arg: The new Entry. cleared: Emitted after the history is cleared. """ - add_completion_item = pyqtSignal(Entry) - item_added = pyqtSignal(Entry) cleared = pyqtSignal() async_read_done = pyqtSignal() def __init__(self, hist_dir, hist_name, parent=None): - super().__init__(parent) + super().__init__("History", ['url', 'title', 'atime', 'redirect'], + primary_key='url', parent=parent) self._initial_read_started = False self._initial_read_done = False self._lineparser = lineparser.AppendLineParser(hist_dir, hist_name, parent=self) - self.history_dict = collections.OrderedDict() - self._temp_history = collections.OrderedDict() + self._temp_history = [] self._new_history = [] self._saved_count = 0 - objreg.get('save-manager').add_saveable( - 'history', self.save, self.item_added) def __repr__(self): return utils.get_repr(self, length=len(self)) - def __iter__(self): - return iter(self.history_dict.values()) - - def __len__(self): - return len(self.history_dict) - def async_read(self): """Read the initial history.""" if self._initial_read_started: @@ -200,21 +179,18 @@ class WebHistory(QObject): self._initial_read_done = True self.async_read_done.emit() + objreg.get('save-manager').add_saveable( + 'history', self.save, self.changed) - for entry in self._temp_history.values(): + for entry in self._temp_history: self._add_entry(entry) self._new_history.append(entry) - if not entry.redirect: - self.add_completion_item.emit(entry) self._temp_history.clear() - def _add_entry(self, entry, target=None): - """Add an entry to self.history_dict or another given OrderedDict.""" - if target is None: - target = self.history_dict - url_str = entry.url_str() - target[url_str] = entry - target.move_to_end(url_str) + def _add_entry(self, entry): + """Add an entry to the in-memory database.""" + self.insert(entry.url_str(), entry.title, entry.atime, entry.redirect, + replace=True) def get_recent(self): """Get the most recent history entries.""" @@ -247,7 +223,7 @@ class WebHistory(QObject): def _do_clear(self): self._lineparser.clear() - self.history_dict.clear() + self.delete_all() self._temp_history.clear() self._new_history.clear() self._saved_count = 0 @@ -291,11 +267,8 @@ class WebHistory(QObject): if self._initial_read_done: self._add_entry(entry) self._new_history.append(entry) - self.item_added.emit(entry) - if not entry.redirect: - self.add_completion_item.emit(entry) else: - self._add_entry(entry, target=self._temp_history) + self._temp_history.append(entry) def init(parent=None): diff --git a/qutebrowser/browser/webkit/webkithistory.py b/qutebrowser/browser/webkit/webkithistory.py index 0f9d64460..453a11883 100644 --- a/qutebrowser/browser/webkit/webkithistory.py +++ b/qutebrowser/browser/webkit/webkithistory.py @@ -48,7 +48,7 @@ class WebHistoryInterface(QWebHistoryInterface): Return: True if the url is in the history, False otherwise. """ - return url_string in self._history.history_dict + return url_string in self._history def init(history): diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index f40e41c2c..6ad461692 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -27,24 +27,33 @@ from hypothesis import strategies from PyQt5.QtCore import QUrl from qutebrowser.browser import history -from qutebrowser.utils import objreg, urlutils, usertypes +from qutebrowser.utils import objreg, urlutils +from qutebrowser.misc import sql -class FakeWebHistory: - - """A fake WebHistory object.""" - - def __init__(self, history_dict): - self.history_dict = history_dict +@pytest.fixture(autouse=True) +def prerequisites(config_stub, fake_save_manager): + """Make sure everything is ready to initialize a WebHistory.""" + config_stub.data = {'general': {'private-browsing': False}} + sql.init() + yield + sql.close() @pytest.fixture() -def hist(tmpdir, fake_save_manager): +def hist(tmpdir): return history.WebHistory(hist_dir=str(tmpdir), hist_name='history') -def test_async_read_twice(monkeypatch, qtbot, tmpdir, caplog, +def test_register_saveable(monkeypatch, qtbot, tmpdir, caplog, fake_save_manager): + (tmpdir / 'filled-history').write('12345 http://example.com/ title') + hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') + list(hist.async_read()) + assert fake_save_manager.add_saveable.called + + +def test_async_read_twice(monkeypatch, qtbot, tmpdir, caplog): (tmpdir / 'filled-history').write('\n'.join([ '12345 http://example.com/ title', '67890 http://example.com/', @@ -55,42 +64,30 @@ def test_async_read_twice(monkeypatch, qtbot, tmpdir, caplog, with pytest.raises(StopIteration): next(hist.async_read()) expected = "Ignoring async_read() because reading is started." - assert len(caplog.records) == 1 - assert caplog.records[0].msg == expected + assert expected in (record.msg for record in caplog.records) @pytest.mark.parametrize('redirect', [True, False]) def test_adding_item_during_async_read(qtbot, hist, redirect): """Check what happens when adding URL while reading the history.""" - url = QUrl('http://www.example.com/') + url = 'http://www.example.com/' + hist.add_url(QUrl(url), redirect=redirect, atime=12345) - with qtbot.assertNotEmitted(hist.add_completion_item), \ - qtbot.assertNotEmitted(hist.item_added): - hist.add_url(url, redirect=redirect, atime=12345) - - if redirect: - with qtbot.assertNotEmitted(hist.add_completion_item): - with qtbot.waitSignal(hist.async_read_done): - list(hist.async_read()) - else: - with qtbot.waitSignals([hist.add_completion_item, - hist.async_read_done], order='strict'): - list(hist.async_read()) + with qtbot.waitSignal(hist.async_read_done): + list(hist.async_read()) assert not hist._temp_history - - expected = history.Entry(url=url, atime=12345, redirect=redirect, title="") - assert list(hist.history_dict.values()) == [expected] + assert list(hist) == [(url, '', 12345, redirect)] def test_iter(hist): list(hist.async_read()) - url = QUrl('http://www.example.com/') + urlstr = 'http://www.example.com/' + url = QUrl(urlstr) hist.add_url(url, atime=12345) - entry = history.Entry(url=url, atime=12345, redirect=False, title="") - assert list(hist) == [entry] + assert list(hist) == [(urlstr, '', 12345, False)] def test_len(hist): @@ -110,38 +107,39 @@ def test_len(hist): ' ', '', ]) -def test_read(hist, tmpdir, line): +def test_read(tmpdir, line): (tmpdir / 'filled-history').write(line + '\n') hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') list(hist.async_read()) -def test_updated_entries(hist, tmpdir): +def test_updated_entries(tmpdir): (tmpdir / 'filled-history').write('12345 http://example.com/\n' '67890 http://example.com/\n') hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') list(hist.async_read()) - assert hist.history_dict['http://example.com/'].atime == 67890 + assert hist['http://example.com/'] == ('http://example.com/', '', 67890, + False) hist.add_url(QUrl('http://example.com/'), atime=99999) - assert hist.history_dict['http://example.com/'].atime == 99999 + assert hist['http://example.com/'] == ('http://example.com/', '', 99999, + False) -def test_invalid_read(hist, tmpdir, caplog): +def test_invalid_read(tmpdir, caplog): (tmpdir / 'filled-history').write('foobar\n12345 http://example.com/') hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') with caplog.at_level(logging.WARNING): list(hist.async_read()) - entries = list(hist.history_dict.values()) + entries = list(hist) assert len(entries) == 1 - assert len(caplog.records) == 1 msg = "Invalid history entry 'foobar': 2 or 3 fields expected!" - assert caplog.records[0].msg == msg + assert msg in (rec.msg for rec in caplog.records) -def test_get_recent(hist, tmpdir): +def test_get_recent(tmpdir): (tmpdir / 'filled-history').write('12345 http://example.com/') hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') list(hist.async_read()) @@ -154,7 +152,7 @@ def test_get_recent(hist, tmpdir): assert lines == expected -def test_save(hist, tmpdir): +def test_save(tmpdir): hist_file = tmpdir / 'filled-history' hist_file.write('12345 http://example.com/\n') @@ -177,7 +175,7 @@ def test_save(hist, tmpdir): assert lines == expected -def test_clear(qtbot, hist, tmpdir): +def test_clear(qtbot, tmpdir): hist_file = tmpdir / 'filled-history' hist_file.write('12345 http://example.com/\n') @@ -190,7 +188,6 @@ def test_clear(qtbot, hist, tmpdir): hist._do_clear() assert not hist_file.read() - assert not hist.history_dict assert not hist._new_history hist.add_url(QUrl('http://www.the-compiler.org/'), atime=67890) @@ -204,24 +201,17 @@ def test_add_item(qtbot, hist): list(hist.async_read()) url = 'http://www.example.com/' - with qtbot.waitSignals([hist.add_completion_item, hist.item_added], - order='strict'): - hist.add_url(QUrl(url), atime=12345, title="the title") + hist.add_url(QUrl(url), atime=12345, title="the title") - entry = history.Entry(url=QUrl(url), redirect=False, atime=12345, - title="the title") - assert hist.history_dict[url] == entry + assert hist[url] == (url, 'the title', 12345, False) def test_add_item_redirect(qtbot, hist): list(hist.async_read()) url = 'http://www.example.com/' - with qtbot.assertNotEmitted(hist.add_completion_item): - with qtbot.waitSignal(hist.item_added): - hist.add_url(QUrl(url), redirect=True, atime=12345) + hist.add_url(QUrl(url), redirect=True, atime=12345) - entry = history.Entry(url=QUrl(url), redirect=True, atime=12345, title="") - assert hist.history_dict[url] == entry + assert hist[url] == (url, '', 12345, True) def test_add_item_redirect_update(qtbot, tmpdir, fake_save_manager): @@ -233,12 +223,9 @@ def test_add_item_redirect_update(qtbot, tmpdir, fake_save_manager): hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') list(hist.async_read()) - with qtbot.assertNotEmitted(hist.add_completion_item): - with qtbot.waitSignal(hist.item_added): - hist.add_url(QUrl(url), redirect=True, atime=67890) + hist.add_url(QUrl(url), redirect=True, atime=67890) - entry = history.Entry(url=QUrl(url), redirect=True, atime=67890, title="") - assert hist.history_dict[url] == entry + assert hist[url] == (url, '', 67890, True) @pytest.mark.parametrize('line, expected', [ @@ -333,8 +320,7 @@ def hist_interface(): entry = history.Entry(atime=0, url=QUrl('http://www.example.com/'), title='example') history_dict = {'http://www.example.com/': entry} - fake_hist = FakeWebHistory(history_dict) - interface = webkithistory.WebHistoryInterface(fake_hist) + interface = webkithistory.WebHistoryInterface(history_dict) QWebHistoryInterface.setDefaultInterface(interface) yield QWebHistoryInterface.setDefaultInterface(None) @@ -349,7 +335,7 @@ def test_history_interface(qtbot, webview, hist_interface): @pytest.mark.parametrize('backend', [usertypes.Backend.QtWebEngine, usertypes.Backend.QtWebKit]) -def test_init(backend, qapp, tmpdir, monkeypatch, fake_save_manager): +def test_init(backend, qapp, tmpdir, monkeypatch): if backend == usertypes.Backend.QtWebKit: pytest.importorskip('PyQt5.QtWebKitWidgets') else: @@ -379,5 +365,4 @@ def test_init(backend, qapp, tmpdir, monkeypatch, fake_save_manager): # before (so we need to test webengine before webkit) assert default_interface is None - assert fake_save_manager.add_saveable.called objreg.delete('web-history') From 93d81d96ce3314866a260d9a1d71b008674944e4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 22 Jan 2017 07:14:42 -0500 Subject: [PATCH 006/337] Use SQL for quickmark/bookmark storage. Store quickmarks and bookmarks in an in-memory sql database instead of a python dict. Long-term storage is not affected, bookmarks and quickmarks are still persisted in a text file. The added and deleted signals were removed, as once sql completion models are used the models will no longer need to update themselves. This will set the stage for SQL-based history completion. See #1765. --- qutebrowser/browser/commands.py | 14 ++-- qutebrowser/browser/urlmarks.py | 112 ++++++++++---------------------- 2 files changed, 42 insertions(+), 84 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 8914d4ac2..9d19914f8 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1260,13 +1260,15 @@ class CommandDispatcher: if name is None: url = self._current_url() try: - name = quickmark_manager.get_by_qurl(url) - except urlmarks.DoesNotExistError as e: + quickmark_manager.delete_by_qurl(url) + except KeyError as e: raise cmdexc.CommandError(str(e)) - try: - quickmark_manager.delete(name) - except KeyError: - raise cmdexc.CommandError("Quickmark '{}' not found!".format(name)) + else: + try: + quickmark_manager.delete(name) + except KeyError: + raise cmdexc.CommandError( + "Quickmark '{}' not found!".format(name)) @cmdutils.register(instance='command-dispatcher', scope='window') def bookmark_add(self, url=None, title=None, toggle=False): diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index 013de408c..db5e14153 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -31,12 +31,12 @@ import os.path import functools import collections -from PyQt5.QtCore import pyqtSignal, QUrl, QObject +from PyQt5.QtCore import QUrl, QObject from qutebrowser.utils import (message, usertypes, qtutils, urlutils, standarddir, objreg, log) from qutebrowser.commands import cmdutils -from qutebrowser.misc import lineparser +from qutebrowser.misc import lineparser, sql class Error(Exception): @@ -53,13 +53,6 @@ class InvalidUrlError(Error): pass -class DoesNotExistError(Error): - - """Exception emitted when a given URL does not exist.""" - - pass - - class AlreadyExistsError(Error): """Exception emitted when a given URL does already exist.""" @@ -67,29 +60,18 @@ class AlreadyExistsError(Error): pass -class UrlMarkManager(QObject): +class UrlMarkManager(sql.SqlTable): """Base class for BookmarkManager and QuickmarkManager. Attributes: marks: An OrderedDict of all quickmarks/bookmarks. _lineparser: The LineParser used for the marks - - Signals: - changed: Emitted when anything changed. - added: Emitted when a new quickmark/bookmark was added. - removed: Emitted when an existing quickmark/bookmark was removed. """ - changed = pyqtSignal() - added = pyqtSignal(str, str) - removed = pyqtSignal(str) - - def __init__(self, parent=None): - """Initialize and read quickmarks.""" - super().__init__(parent) - - self.marks = collections.OrderedDict() + def __init__(self, name, fields, primary_key, parent=None): + """Initialize and read marks.""" + super().__init__(name, fields, primary_key, parent) self._init_lineparser() for line in self._lineparser: @@ -110,32 +92,21 @@ class UrlMarkManager(QObject): def save(self): """Save the marks to disk.""" - self._lineparser.data = [' '.join(tpl) for tpl in self.marks.items()] + self._lineparser.data = [' '.join(tpl) for tpl in self] self._lineparser.save() - def delete(self, key): - """Delete a quickmark/bookmark. - - Args: - key: The key to delete (name for quickmarks, URL for bookmarks.) - """ - del self.marks[key] - self.changed.emit() - self.removed.emit(key) - class QuickmarkManager(UrlMarkManager): """Manager for quickmarks. - The primary key for quickmarks is their *name*, this means: - - - self.marks maps names to URLs. - - changed gets emitted with the name as first argument and the URL as - second argument. - - removed gets emitted with the name as argument. + The primary key for quickmarks is their *name*. """ + def __init__(self, parent=None): + super().__init__('Quickmarks', ['name', 'url'], primary_key='name', + parent=parent) + def _init_lineparser(self): self._lineparser = lineparser.LineParser( standarddir.config(), 'quickmarks', parent=self) @@ -151,7 +122,7 @@ class QuickmarkManager(UrlMarkManager): except ValueError: message.error("Invalid quickmark '{}'".format(line)) else: - self.marks[key] = url + self.insert(key, url) def prompt_save(self, url): """Prompt for a new quickmark name to be added and add it. @@ -191,41 +162,30 @@ class QuickmarkManager(UrlMarkManager): def set_mark(): """Really set the quickmark.""" - self.marks[name] = url - self.changed.emit() - self.added.emit(name, url) + self.insert(name, url) log.misc.debug("Added quickmark {} for {}".format(name, url)) - if name in self.marks: + if name in self: message.confirm_async( title="Override existing quickmark?", yes_action=set_mark, default=True) else: set_mark() - def get_by_qurl(self, url): - """Look up a quickmark by QUrl, returning its name. - - Takes O(n) time, where n is the number of quickmarks. - Use a name instead where possible. - """ + def delete_by_qurl(self, url): + """Look up a quickmark by QUrl, returning its name.""" qtutils.ensure_valid(url) urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) - try: - index = list(self.marks.values()).index(urlstr) - key = list(self.marks.keys())[index] - except ValueError: - raise DoesNotExistError( - "Quickmark for '{}' not found!".format(urlstr)) - return key + self.delete(urlstr, field='url') + except KeyError: + raise KeyError("Quickmark for '{}' not found!".format(urlstr)) def get(self, name): """Get the URL of the quickmark named name as a QUrl.""" - if name not in self.marks: - raise DoesNotExistError( - "Quickmark '{}' does not exist!".format(name)) - urlstr = self.marks[name] + if name not in self: + raise KeyError("Quickmark '{}' does not exist!".format(name)) + urlstr = self[name] try: url = urlutils.fuzzy_url(urlstr, do_search=False) except urlutils.InvalidUrlError as e: @@ -238,14 +198,13 @@ class BookmarkManager(UrlMarkManager): """Manager for bookmarks. - The primary key for bookmarks is their *url*, this means: - - - self.marks maps URLs to titles. - - changed gets emitted with the URL as first argument and the title as - second argument. - - removed gets emitted with the URL as argument. + The primary key for bookmarks is their *url*. """ + def __init__(self, parent=None): + super().__init__('Bookmarks', ['url', 'title'], primary_key='url', + parent=parent) + def _init_lineparser(self): bookmarks_directory = os.path.join(standarddir.config(), 'bookmarks') if not os.path.isdir(bookmarks_directory): @@ -262,10 +221,9 @@ class BookmarkManager(UrlMarkManager): def _parse_line(self, line): parts = line.split(maxsplit=1) - if len(parts) == 2: - self.marks[parts[0]] = parts[1] - elif len(parts) == 1: - self.marks[parts[0]] = '' + urlstr = parts[0] + title = parts[1] if len(parts) == 2 else '' + self.insert(urlstr, title) def add(self, url, title, *, toggle=False): """Add a new bookmark. @@ -286,14 +244,12 @@ class BookmarkManager(UrlMarkManager): urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) - if urlstr in self.marks: + if urlstr in self: if toggle: - del self.marks[urlstr] + self.delete(urlstr) return False else: raise AlreadyExistsError("Bookmark already exists!") else: - self.marks[urlstr] = title - self.changed.emit() - self.added.emit(title, urlstr) + self.insert(urlstr, title) return True From 6e1ea89ca17b0dc0a77ffcfd74f359ef1337088d Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 22 Jan 2017 07:18:41 -0500 Subject: [PATCH 007/337] Implement SQL completion model. This model wraps one or more SQL tables and exposes data in a tiered manner that can be consumed by the tree view used for completion. --- qutebrowser/completion/completer.py | 4 +- qutebrowser/completion/models/sqlmodel.py | 218 ++++++++++++++++++++++ tests/unit/completion/test_sqlmodel.py | 204 ++++++++++++++++++++ 3 files changed, 424 insertions(+), 2 deletions(-) create mode 100644 qutebrowser/completion/models/sqlmodel.py create mode 100644 tests/unit/completion/test_sqlmodel.py diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index bd4f43009..8742156a3 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -73,8 +73,8 @@ class Completer(QObject): A completion model or None. """ model = completion(*pos_args) - if model is None: - return None + if model is None or hasattr(model, 'set_pattern'): + return model else: return sortfilter.CompletionFilterModel(source=model, parent=self) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py new file mode 100644 index 000000000..a5a9bc99a --- /dev/null +++ b/qutebrowser/completion/models/sqlmodel.py @@ -0,0 +1,218 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""A completion model backed by SQL tables.""" + +import re + +from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel +from PyQt5.QtSql import QSqlTableModel, QSqlDatabase, QSqlQuery + +from qutebrowser.utils import usertypes, log + + +class SqlCompletionModel(QAbstractItemModel): + + """A sqlite-based model that provides data for the CompletionView. + + This model is a wrapper around one or more sql tables. The tables are all + stored in a single database in qutebrowser's cache directory. + + Top level indices represent categories, each of which is backed by a single + table. Child indices represent rows of those tables. + + Class Attributes: + COLUMN_WIDTHS: The width percentages of the columns used in the + completion view. + + Attributes: + column_widths: The width percentages of the columns used in the + completion view. + columns_to_filter: A list of indices of columns to apply the filter to. + pattern: Current filter pattern, used for highlighting. + _categories: The category tables. + """ + + def __init__(self, column_widths=(30, 70, 0), columns_to_filter=None, + parent=None): + super().__init__(parent) + self.columns_to_filter = columns_to_filter or [0] + self.column_widths = column_widths + self._categories = [] + self.srcmodel = self # TODO: dummy for compat with old API + self.pattern = '' + + def new_category(self, name, sort_by=None, sort_order=Qt.AscendingOrder): + """Create a new completion category and add it to this model. + + Args: + name: Name of category, and the table in the database. + sort_by: The name of the field to sort by, or None for no sorting. + sort_order: Sorting order, if sort_by is non-None. + + Return: A new CompletionCategory. + """ + database = QSqlDatabase.database() + cat = QSqlTableModel(parent=self, db=database) + cat.setTable(name) + if sort_by: + cat.setSort(cat.fieldIndex(sort_by), sort_order) + cat.select() + self._categories.append(cat) + return cat + + def delete_cur_item(self, completion): + """Delete the selected item.""" + raise NotImplementedError + + def data(self, index, role=Qt.DisplayRole): + """Return the item data for index. + + Override QAbstractItemModel::data. + + Args: + index: The QModelIndex to get item flags for. + + Return: The item data, or None on an invalid index. + """ + if not index.isValid() or role != Qt.DisplayRole: + return + if not index.parent().isValid(): + if index.column() == 0: + return self._categories[index.row()].tableName() + else: + table = self._categories[index.parent().row()] + idx = table.index(index.row(), index.column()) + return table.data(idx) + + def flags(self, index): + """Return the item flags for index. + + Override QAbstractItemModel::flags. + + Return: The item flags, or Qt.NoItemFlags on error. + """ + if not index.isValid(): + return + if index.parent().isValid(): + # item + return (Qt.ItemIsEnabled | Qt.ItemIsSelectable | + Qt.ItemNeverHasChildren) + else: + # category + return Qt.NoItemFlags + + def index(self, row, col, parent=QModelIndex()): + """Get an index into the model. + + Override QAbstractItemModel::index. + + Return: A QModelIndex. + """ + if (row < 0 or row >= self.rowCount(parent) or + col < 0 or col >= self.columnCount(parent)): + return QModelIndex() + if parent.isValid(): + if parent.column() != 0: + return QModelIndex() + # store a pointer to the parent table in internalPointer + return self.createIndex(row, col, self._categories[parent.row()]) + return self.createIndex(row, col, None) + + def parent(self, index): + """Get an index to the parent of the given index. + + Override QAbstractItemModel::parent. + + Args: + index: The QModelIndex to get the parent index for. + """ + parent_table = index.internalPointer() + if not parent_table: + # categories have no parent + return QModelIndex() + row = self._categories.index(parent_table) + return self.createIndex(row, 0, None) + + def rowCount(self, parent=QModelIndex()): + if not parent.isValid(): + # top-level + return len(self._categories) + elif parent.internalPointer() or parent.column() != 0: + # item or nonzero category column (only first col has children) + return 0 + else: + # category + return self._categories[parent.row()].rowCount() + + def columnCount(self, parent=QModelIndex()): + # pylint: disable=unused-argument + return 3 + + def count(self): + """Return the count of non-category items.""" + return sum(t.rowCount() for t in self._categories) + + def set_pattern(self, pattern): + """Set the filter pattern for all category tables. + + This will apply to the fields indicated in columns_to_filter. + + Args: + pattern: The filter pattern to set. + """ + # TODO: should pattern be saved in the view layer instead? + self.pattern = pattern + # escape to treat a user input % or _ as a literal, not a wildcard + pattern = pattern.replace('%', '\\%') + pattern = pattern.replace('_', '\\_') + # treat spaces as wildcards to match any of the typed words + pattern = re.sub(r' +', '%', pattern) + for t in self._categories: + fields = (t.record().fieldName(i) for i in self.columns_to_filter) + query = ' or '.join("{} like '%{}%' escape '\\'" + .format(field, pattern) + for field in fields) + log.completion.debug("Setting filter = '{}' for table '{}'" + .format(query, t.tableName())) + t.setFilter(query) + + def first_item(self): + """Return the index of the first child (non-category) in the model.""" + for row, table in enumerate(self._categories): + if table.rowCount() > 0: + parent = self.index(row, 0) + return self.index(0, 0, parent) + return QModelIndex() + + def last_item(self): + """Return the index of the last child (non-category) in the model.""" + for row, table in reversed(list(enumerate(self._categories))): + childcount = table.rowCount() + if childcount > 0: + parent = self.index(row, 0) + return self.index(childcount - 1, 0, parent) + return QModelIndex() + + +class SqlException(Exception): + + """Raised on an error interacting with the SQL database.""" + + pass diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py new file mode 100644 index 000000000..6b8f01f4d --- /dev/null +++ b/tests/unit/completion/test_sqlmodel.py @@ -0,0 +1,204 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Tests for the base sql completion model.""" + +import pytest +from PyQt5.QtCore import Qt + +from qutebrowser.misc import sql +from qutebrowser.completion.models import sqlmodel + + +@pytest.fixture(autouse=True) +def init(): + sql.init() + yield + sql.close() + + +def _check_model(model, expected): + """Check that a model contains the expected items in the given order. + + Args: + expected: A list of form + [ + (cat, [(name, desc, misc), (name, desc, misc), ...]), + (cat, [(name, desc, misc), (name, desc, misc), ...]), + ... + ] + """ + assert model.rowCount() == len(expected) + for i, (expected_title, expected_items) in enumerate(expected): + catidx = model.index(i, 0) + assert model.data(catidx) == expected_title + assert model.rowCount(catidx) == len(expected_items) + for j, (name, desc, misc) in enumerate(expected_items): + assert model.data(model.index(j, 0, catidx)) == name + assert model.data(model.index(j, 1, catidx)) == desc + assert model.data(model.index(j, 2, catidx)) == misc + + +@pytest.mark.parametrize('rowcounts, expected', [ + ([0], 0), + ([1], 1), + ([2], 2), + ([0, 0], 0), + ([0, 0, 0], 0), + ([1, 1], 2), + ([3, 2, 1], 6), + ([0, 2, 0], 2), +]) +def test_count(rowcounts, expected): + model = sqlmodel.SqlCompletionModel() + for i, rowcount in enumerate(rowcounts): + name = 'Foo' + str(i) + table = sql.SqlTable(name, ['a'], primary_key='a') + for rownum in range(rowcount): + table.insert(rownum) + model.new_category(name) + assert model.count() == expected + + +@pytest.mark.parametrize('sort_by, sort_order, data, expected', [ + (None, Qt.AscendingOrder, + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), + + ('a', Qt.AscendingOrder, + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), + + ('a', Qt.DescendingOrder, + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), + + ('b', Qt.AscendingOrder, + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), + + ('b', Qt.DescendingOrder, + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), + + ('c', Qt.AscendingOrder, + [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], + [('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]), + + ('c', Qt.DescendingOrder, + [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], + [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), +]) +def test_sorting(sort_by, sort_order, data, expected): + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + for row in data: + table.insert(*row) + model = sqlmodel.SqlCompletionModel() + model.new_category('Foo', sort_by=sort_by, sort_order=sort_order) + _check_model(model, [('Foo', expected)]) + + +@pytest.mark.parametrize('pattern, filter_cols, before, after', [ + ('foo', [0], + [('A', [('foo', '', ''), ('bar', '', ''), ('aafobbb', '', '')])], + [('A', [('foo', '', '')])]), + + ('foo', [0], + [('A', [('baz', 'bar', 'foo'), ('foo', '', ''), ('bar', 'foo', '')])], + [('A', [('foo', '', '')])]), + + ('foo', [0], + [('A', [('foo', '', ''), ('bar', '', '')]), + ('B', [('foo', '', ''), ('bar', '', '')])], + [('A', [('foo', '', '')]), ('B', [('foo', '', '')])]), + + ('foo', [0], + [('A', [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')])], + [('A', [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')])]), + + ('foo', [0], + [('A', [('foo', '', '')]), ('B', [('bar', '', '')])], + [('A', [('foo', '', '')]), ('B', [])]), + + ('foo', [1], + [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])], + [('A', [('bar', 'foo', '')])]), + + ('foo', [0, 1], + [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])], + [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])]), + + ('foo', [0, 1, 2], + [('A', [('foo', '', ''), ('bar', '', '')])], + [('A', [('foo', '', '')])]), + + ('foo bar', [0], + [('A', [('foo', '', ''), ('bar foo', '', ''), ('xfooyybarz', '', '')])], + [('A', [('xfooyybarz', '', '')])]), + + ('foo%bar', [0], + [('A', [('foo%bar', '', ''), ('foo bar', '', ''), ('foobar', '', '')])], + [('A', [('foo%bar', '', '')])]), + + ('_', [0], + [('A', [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')])], + [('A', [('a_b', '', ''), ('__a', '', '')])]), +]) +def test_set_pattern(pattern, filter_cols, before, after): + """Validate the filtering and sorting results of set_pattern.""" + model = sqlmodel.SqlCompletionModel() + for name, rows in before: + table = sql.SqlTable(name, ['a', 'b', 'c'], primary_key='a') + for row in rows: + table.insert(*row) + model.new_category(name) + model.columns_to_filter = filter_cols + model.set_pattern(pattern) + _check_model(model, after) + + +@pytest.mark.parametrize('data, first, last', [ + ([('A', ['Aa'])], 'Aa', 'Aa'), + ([('A', ['Aa', 'Ba'])], 'Aa', 'Ba'), + ([('A', ['Aa', 'Ab', 'Ac']), ('B', ['Ba', 'Bb']), + ('C', ['Ca'])], 'Aa', 'Ca'), + ([('A', []), ('B', ['Ba'])], 'Ba', 'Ba'), + ([('A', []), ('B', []), ('C', ['Ca'])], 'Ca', 'Ca'), + ([('A', []), ('B', []), ('C', ['Ca', 'Cb'])], 'Ca', 'Cb'), + ([('A', ['Aa']), ('B', [])], 'Aa', 'Aa'), + ([('A', ['Aa']), ('B', []), ('C', [])], 'Aa', 'Aa'), + ([('A', ['Aa']), ('B', []), ('C', ['Ca'])], 'Aa', 'Ca'), + ([('A', []), ('B', [])], None, None), +]) +def test_first_last_item(data, first, last): + """Test that first() and last() return indexes to the first and last items. + + Args: + data: Input to _make_model + first: text of the first item + last: text of the last item + """ + model = sqlmodel.SqlCompletionModel() + for name, rows in data: + table = sql.SqlTable(name, ['a'], primary_key='a') + for row in rows: + table.insert(row) + model.new_category(name) + assert model.data(model.first_item()) == first + assert model.data(model.last_item()) == last From acea0d3c671fb3dc6e8c521710d0c89892b87453 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 5 Feb 2017 19:34:14 -0500 Subject: [PATCH 008/337] Use SQL completion for the open command. Now that history, bookmark, and quickmark storage are SQL-backed, use a sql completion model to serve url completions. --- qutebrowser/completion/models/urlmodel.py | 183 ++-------------------- tests/unit/browser/webkit/test_history.py | 2 +- 2 files changed, 12 insertions(+), 173 deletions(-) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index d8b7f42a2..bd3efd469 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -19,180 +19,19 @@ """Function to return the url completion model for the `open` command.""" -import datetime - -from PyQt5.QtCore import pyqtSlot, Qt - -from qutebrowser.utils import objreg, utils, qtutils, log -from qutebrowser.completion.models import base -from qutebrowser.config import config - -_URL_COLUMN = 0 -_TEXT_COLUMN = 1 -_TIME_COLUMN = 2 -_model = None -_history_cat = None -_quickmark_cat = None -_bookmark_cat = None - - -def _delete_url(completion): - index = completion.currentIndex() - qtutils.ensure_valid(index) - category = index.parent() - index = category.child(index.row(), _URL_COLUMN) - qtutils.ensure_valid(category) - if category.data() == 'Bookmarks': - bookmark_manager = objreg.get('bookmark-manager') - bookmark_manager.delete(index.data()) - elif category.data() == 'Quickmarks': - quickmark_manager = objreg.get('quickmark-manager') - sibling = index.sibling(index.row(), _TEXT_COLUMN) - qtutils.ensure_valid(sibling) - name = sibling.data() - quickmark_manager.delete(name) - - -def _remove_oldest_history(): - """Remove the oldest history entry.""" - _history_cat.removeRow(0) - - -def _add_history_entry(entry): - """Add a new history entry to the completion.""" - _model.new_item(_history_cat, entry.url.toDisplayString(), - entry.title, _fmt_atime(entry.atime), - sort=int(entry.atime), userdata=entry.url) - - max_history = config.get('completion', 'web-history-max-items') - if max_history != -1 and _history_cat.rowCount() > max_history: - _remove_oldest_history() - - -@config.change_filter('completion', 'timestamp-format') -def _reformat_timestamps(): - """Reformat the timestamps if the config option was changed.""" - for i in range(_history_cat.rowCount()): - url_item = _history_cat.child(i, _URL_COLUMN) - atime_item = _history_cat.child(i, _TIME_COLUMN) - atime = url_item.data(base.Role.sort) - atime_item.setText(_fmt_atime(atime)) - - -@pyqtSlot(object) -def _on_history_item_added(entry): - """Slot called when a new history item was added.""" - for i in range(_history_cat.rowCount()): - url_item = _history_cat.child(i, _URL_COLUMN) - atime_item = _history_cat.child(i, _TIME_COLUMN) - title_item = _history_cat.child(i, _TEXT_COLUMN) - if url_item.data(base.Role.userdata) == entry.url: - atime_item.setText(_fmt_atime(entry.atime)) - title_item.setText(entry.title) - url_item.setData(int(entry.atime), base.Role.sort) - break - else: - _add_history_entry(entry) - - -@pyqtSlot() -def _on_history_cleared(): - _history_cat.removeRows(0, _history_cat.rowCount()) - - -def _remove_item(data, category, column): - """Helper function for on_quickmark_removed and on_bookmark_removed. - - Args: - data: The item to search for. - category: The category to search in. - column: The column to use for matching. - """ - for i in range(category.rowCount()): - item = category.child(i, column) - if item.data(Qt.DisplayRole) == data: - category.removeRow(i) - break - - -@pyqtSlot(str) -def _on_quickmark_removed(name): - """Called when a quickmark has been removed by the user. - - Args: - name: The name of the quickmark which has been removed. - """ - _remove_item(name, _quickmark_cat, _TEXT_COLUMN) - - -@pyqtSlot(str) -def _on_bookmark_removed(urltext): - """Called when a bookmark has been removed by the user. - - Args: - urltext: The url of the bookmark which has been removed. - """ - _remove_item(urltext, _bookmark_cat, _URL_COLUMN) - - -def _fmt_atime(atime): - """Format an atime to a human-readable string.""" - fmt = config.get('completion', 'timestamp-format') - if fmt is None: - return '' - try: - dt = datetime.datetime.fromtimestamp(atime) - except (ValueError, OSError, OverflowError): - # Different errors which can occur for too large values... - log.misc.error("Got invalid timestamp {}!".format(atime)) - return '(invalid)' - else: - return dt.strftime(fmt) - - -def _init(): - global _model, _quickmark_cat, _bookmark_cat, _history_cat - _model = base.CompletionModel(column_widths=(40, 50, 10), - dumb_sort=Qt.DescendingOrder, - delete_cur_item=_delete_url, - columns_to_filter=[_URL_COLUMN, - _TEXT_COLUMN]) - _quickmark_cat = _model.new_category("Quickmarks") - _bookmark_cat = _model.new_category("Bookmarks") - _history_cat = _model.new_category("History") - - quickmark_manager = objreg.get('quickmark-manager') - quickmarks = quickmark_manager.marks.items() - for qm_name, qm_url in quickmarks: - _model.new_item(_quickmark_cat, qm_url, qm_name) - quickmark_manager.added.connect( - lambda name, url: _model.new_item(_quickmark_cat, url, name)) - quickmark_manager.removed.connect(_on_quickmark_removed) - - bookmark_manager = objreg.get('bookmark-manager') - bookmarks = bookmark_manager.marks.items() - for bm_url, bm_title in bookmarks: - _model.new_item(_bookmark_cat, bm_url, bm_title) - bookmark_manager.added.connect( - lambda name, url: _model.new_item(_bookmark_cat, url, name)) - bookmark_manager.removed.connect(_on_bookmark_removed) - - history = objreg.get('web-history') - max_history = config.get('completion', 'web-history-max-items') - for entry in utils.newest_slice(history, max_history): - if not entry.redirect: - _add_history_entry(entry) - history.add_completion_item.connect(_on_history_item_added) - history.cleared.connect(_on_history_cleared) - - objreg.get('config').changed.connect(_reformat_timestamps) - +from qutebrowser.completion.models import sqlmodel def url(): - """A _model which combines bookmarks, quickmarks and web history URLs. + """A model which combines bookmarks, quickmarks and web history URLs. Used for the `open` command. """ - if not _model: - _init() - return _model + urlcol = 0 + textcol = 1 + + model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), + columns_to_filter=[urlcol, textcol]) + model.new_category('History') + model.new_category('Quickmarks') + model.new_category('Bookmarks') + return model diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 6ad461692..de7c27a77 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -27,7 +27,7 @@ from hypothesis import strategies from PyQt5.QtCore import QUrl from qutebrowser.browser import history -from qutebrowser.utils import objreg, urlutils +from qutebrowser.utils import objreg, urlutils, usertypes from qutebrowser.misc import sql From 3005374ada236d8b126e2194881663401edb654c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 8 Feb 2017 17:21:27 -0500 Subject: [PATCH 009/337] Simplify sorting logic in sortfilter. For URL completion, time-based sorting is handled by the SQL model. All the other models use simple alphabetical sorting. This allowed cleaning up some logic in the sortfilter, removing DUMB_SORT, and removing the completion.Role.sort. This also removes the userdata completion field as it was only used in url completion and is no longer necessary with the SQL model. --- qutebrowser/completion/models/base.py | 26 ++-------- qutebrowser/completion/models/configmodel.py | 4 +- qutebrowser/completion/models/miscmodels.py | 1 - qutebrowser/completion/models/sortfilter.py | 28 ++-------- tests/unit/completion/test_completer.py | 1 - tests/unit/completion/test_sortfilter.py | 54 ++++---------------- 6 files changed, 19 insertions(+), 95 deletions(-) diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py index 43f3a1b48..1d3ed2be4 100644 --- a/qutebrowser/completion/models/base.py +++ b/qutebrowser/completion/models/base.py @@ -29,10 +29,6 @@ from PyQt5.QtGui import QStandardItemModel, QStandardItem from qutebrowser.utils import usertypes -Role = usertypes.enum('Role', ['sort', 'userdata'], start=Qt.UserRole, - is_int=True) - - class CompletionModel(QStandardItemModel): """A simple QStandardItemModel adopted for completions. @@ -42,37 +38,30 @@ class CompletionModel(QStandardItemModel): Attributes: column_widths: The width percentages of the columns used in the - completion view. - dumb_sort: the dumb sorting used by the model """ - def __init__(self, dumb_sort=None, column_widths=(30, 70, 0), - columns_to_filter=None, delete_cur_item=None, parent=None): + def __init__(self, column_widths=(30, 70, 0), columns_to_filter=None, + delete_cur_item=None, parent=None): super().__init__(parent) self.setColumnCount(3) self.columns_to_filter = columns_to_filter or [0] - self.dumb_sort = dumb_sort self.column_widths = column_widths self.delete_cur_item = delete_cur_item - def new_category(self, name, sort=None): + def new_category(self, name): """Add a new category to the model. Args: name: The name of the category to add. - sort: The value to use for the sort role. Return: The created QStandardItem. """ cat = QStandardItem(name) - if sort is not None: - cat.setData(sort, Role.sort) self.appendRow(cat) return cat - def new_item(self, cat, name, desc='', misc=None, sort=None, - userdata=None): + def new_item(self, cat, name, desc='', misc=None): """Add a new item to a category. Args: @@ -80,8 +69,6 @@ class CompletionModel(QStandardItemModel): name: The name of the item. desc: The description of the item. misc: Misc text to display. - sort: Data for the sort role (int). - userdata: User data to be added for the first column. Return: A (nameitem, descitem, miscitem) tuple. @@ -98,11 +85,6 @@ class CompletionModel(QStandardItemModel): miscitem = QStandardItem(misc) cat.appendRow([nameitem, descitem, miscitem]) - if sort is not None: - nameitem.setData(sort, Role.sort) - if userdata is not None: - nameitem.setData(userdata, Role.userdata) - return nameitem, descitem, miscitem def flags(self, index): """Return the item flags for index. diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index 5ed2a47d0..406d8d572 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -67,7 +67,7 @@ def value(sectname, optname): optname: The name of the config option this model shows. """ model = base.CompletionModel(column_widths=(20, 70, 10)) - cur_cat = model.new_category("Current/Default", sort=0) + cur_cat = model.new_category("Current/Default") try: val = config.get(sectname, optname, raw=True) or '""' except (configexc.NoSectionError, configexc.NoOptionError): @@ -85,7 +85,7 @@ def value(sectname, optname): # Different type for each value (KeyValue) vals = configdata.DATA[sectname][optname].typ.complete() if vals is not None: - cat = model.new_category("Completions", sort=1) + cat = model.new_category("Completions") for (val, desc) in vals: model.new_item(cat, val, desc) return model diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index bcbb94177..cb87f9d7e 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -119,7 +119,6 @@ def buffer(): model = base.CompletionModel( column_widths=(6, 40, 54), - dumb_sort=Qt.DescendingOrder, delete_cur_item=delete_buffer, columns_to_filter=[idx_column, url_column, text_column]) diff --git a/qutebrowser/completion/models/sortfilter.py b/qutebrowser/completion/models/sortfilter.py index 92dd1b2a0..e5d42607c 100644 --- a/qutebrowser/completion/models/sortfilter.py +++ b/qutebrowser/completion/models/sortfilter.py @@ -33,14 +33,13 @@ from qutebrowser.completion.models import base as completion class CompletionFilterModel(QSortFilterProxyModel): - """Subclass of QSortFilterProxyModel with custom sorting/filtering. + """Subclass of QSortFilterProxyModel with custom filtering. Attributes: pattern: The pattern to filter with. srcmodel: The current source model. Kept as attribute because calling `sourceModel` takes quite a long time for some reason. - _sort_order: The order to use for sorting if using dumb_sort. """ def __init__(self, source, parent=None): @@ -49,21 +48,12 @@ class CompletionFilterModel(QSortFilterProxyModel): self.srcmodel = source self.pattern = '' self.pattern_re = None - - dumb_sort = self.srcmodel.dumb_sort - if dumb_sort is None: - # pylint: disable=invalid-name - self.lessThan = self.intelligentLessThan - self._sort_order = Qt.AscendingOrder - else: - self.setSortRole(completion.Role.sort) - self._sort_order = dumb_sort + self.lessThan = self.intelligentLessThan + #self._sort_order = self.srcmodel.sort_order or Qt.AscendingOrder def set_pattern(self, val): """Setter for pattern. - Invalidates the filter and re-sorts the model. - Args: val: The value to set. """ @@ -163,12 +153,6 @@ class CompletionFilterModel(QSortFilterProxyModel): qtutils.ensure_valid(lindex) qtutils.ensure_valid(rindex) - left_sort = self.srcmodel.data(lindex, role=completion.Role.sort) - right_sort = self.srcmodel.data(rindex, role=completion.Role.sort) - - if left_sort is not None and right_sort is not None: - return left_sort < right_sort - left = self.srcmodel.data(lindex) right = self.srcmodel.data(rindex) @@ -183,9 +167,3 @@ class CompletionFilterModel(QSortFilterProxyModel): return False else: return left < right - - def sort(self, column, order=None): - """Extend sort to respect self._sort_order if no order was given.""" - if order is None: - order = self._sort_order - super().sort(column, order) diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index 48f619104..34c9a2d19 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -37,7 +37,6 @@ class FakeCompletionModel(QStandardItemModel): super().__init__(parent) self.kind = kind self.pos_args = [*pos_args] - self.dumb_sort = None class CompletionWidgetStub(QObject): diff --git a/tests/unit/completion/test_sortfilter.py b/tests/unit/completion/test_sortfilter.py index c972a5ec9..7d84e9489 100644 --- a/tests/unit/completion/test_sortfilter.py +++ b/tests/unit/completion/test_sortfilter.py @@ -151,80 +151,46 @@ def test_count(tree, expected): assert filter_model.count() == expected -@pytest.mark.parametrize('pattern, dumb_sort, filter_cols, before, after', [ - ('foo', None, [0], +@pytest.mark.parametrize('pattern, filter_cols, before, after', [ + ('foo', [0], [[('foo', '', ''), ('bar', '', '')]], [[('foo', '', '')]]), - ('foo', None, [0], + ('foo', [0], [[('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')]], [[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]]), - ('foo', None, [0], + ('foo', [0], [[('foo', '', '')], [('bar', '', '')]], [[('foo', '', '')], []]), # prefer foobar as it starts with the pattern - ('foo', None, [0], + ('foo', [0], [[('barfoo', '', ''), ('foobar', '', '')]], [[('foobar', '', ''), ('barfoo', '', '')]]), # however, don't rearrange categories - ('foo', None, [0], + ('foo', [0], [[('barfoo', '', '')], [('foobar', '', '')]], [[('barfoo', '', '')], [('foobar', '', '')]]), - ('foo', None, [1], + ('foo', [1], [[('foo', 'bar', ''), ('bar', 'foo', '')]], [[('bar', 'foo', '')]]), - ('foo', None, [0, 1], + ('foo', [0, 1], [[('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')]], [[('foo', 'bar', ''), ('bar', 'foo', '')]]), - ('foo', None, [0, 1, 2], + ('foo', [0, 1, 2], [[('foo', '', ''), ('bar', '')]], [[('foo', '', '')]]), - - # the fourth column is the sort role, which overrides data-based sorting - ('', None, [0], - [[('two', '', '', 2), ('one', '', '', 1), ('three', '', '', 3)]], - [[('one', '', ''), ('two', '', ''), ('three', '', '')]]), - - ('', Qt.AscendingOrder, [0], - [[('two', '', '', 2), ('one', '', '', 1), ('three', '', '', 3)]], - [[('one', '', ''), ('two', '', ''), ('three', '', '')]]), - - ('', Qt.DescendingOrder, [0], - [[('two', '', '', 2), ('one', '', '', 1), ('three', '', '', 3)]], - [[('three', '', ''), ('two', '', ''), ('one', '', '')]]), ]) -def test_set_pattern(pattern, dumb_sort, filter_cols, before, after): +def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" model = _create_model(before) - model.dumb_sort = dumb_sort model.columns_to_filter = filter_cols filter_model = sortfilter.CompletionFilterModel(model) filter_model.set_pattern(pattern) actual = _extract_model_data(filter_model) assert actual == after - - -def test_sort(): - """Ensure that a sort argument passed to sort overrides DUMB_SORT. - - While test_set_pattern above covers most of the sorting logic, this - particular case is easier to test separately. - """ - model = _create_model([[('B', '', '', 1), - ('C', '', '', 2), - ('A', '', '', 0)]]) - filter_model = sortfilter.CompletionFilterModel(model) - - filter_model.sort(0, Qt.AscendingOrder) - actual = _extract_model_data(filter_model) - assert actual == [[('A', '', ''), ('B', '', ''), ('C', '', '')]] - - filter_model.sort(0, Qt.DescendingOrder) - actual = _extract_model_data(filter_model) - assert actual == [[('C', '', ''), ('B', '', ''), ('A', '', '')]] From 93f89849871c64e21536d2b47355c89e55c37eaa Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 27 Nov 2016 16:52:03 -0500 Subject: [PATCH 010/337] Install pyqt5.qtsql bindings for debian CI. SQL is included in the Archlinux pyqt5 package, but not in Debian. We need this so the debian-based CI builds will pass with the new sql-based completion implementation. --- scripts/dev/ci/travis_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index 0ada134c1..aa4c1c6c8 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -102,7 +102,7 @@ elif [[ $TRAVIS_OS_NAME == osx ]]; then exit 0 fi -pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit" +pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit python3-pyqt5.qtsql" pip_install pip pip_install -r misc/requirements/requirements-tox.txt From d4f2a70f83a2ee6fbbb580a81fc22094e0f354fe Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 08:36:50 -0500 Subject: [PATCH 011/337] Slightly simplify CompletionModel.new_item. There was no need to have a branch based on whether the misc value was None or not. --- qutebrowser/completion/models/base.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py index 1d3ed2be4..5e2c78c5a 100644 --- a/qutebrowser/completion/models/base.py +++ b/qutebrowser/completion/models/base.py @@ -79,10 +79,7 @@ class CompletionModel(QStandardItemModel): nameitem = QStandardItem(name) descitem = QStandardItem(desc) - if misc is None: - miscitem = QStandardItem() - else: - miscitem = QStandardItem(misc) + miscitem = QStandardItem(misc) cat.appendRow([nameitem, descitem, miscitem]) From a774647c260b1a9d920c8699a60aa0a992b814bf Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 08:39:24 -0500 Subject: [PATCH 012/337] Get test_models mostly working again. - Adjust _check_completions to work for CompletionModel and SqlCompletionModel - Move sql initialization into a reusable fixture - Remove the bookmark/quickmark/history stubs, as they're now handled by sql - Disable quickmark/bookmark model tests until their completion is ported to sql. - Disable urlmodel tests for features that have to be implemented in SQL: - LIMIT (for history-max-items) - Configurable column order (for quickmarks) - Configurable formatting (for timestamp-format --- tests/helpers/fixtures.py | 37 ++----- tests/unit/browser/webkit/test_history.py | 5 +- tests/unit/completion/test_models.py | 114 ++++++++++++---------- tests/unit/completion/test_sqlmodel.py | 6 +- tests/unit/misc/test_sql.py | 6 +- 5 files changed, 76 insertions(+), 92 deletions(-) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 3af7195f5..7fc05a578 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -41,7 +41,7 @@ import helpers.stubs as stubsmod from qutebrowser.config import config from qutebrowser.utils import objreg, standarddir from qutebrowser.browser.webkit import cookies -from qutebrowser.misc import savemanager +from qutebrowser.misc import savemanager, sql from qutebrowser.keyinput import modeman from PyQt5.QtCore import PYQT_VERSION, pyqtSignal, QEvent, QSize, Qt, QObject @@ -239,33 +239,6 @@ def host_blocker_stub(stubs): objreg.delete('host-blocker') -@pytest.fixture -def quickmark_manager_stub(stubs): - """Fixture which provides a fake quickmark manager object.""" - stub = stubs.QuickmarkManagerStub() - objreg.register('quickmark-manager', stub) - yield stub - objreg.delete('quickmark-manager') - - -@pytest.fixture -def bookmark_manager_stub(stubs): - """Fixture which provides a fake bookmark manager object.""" - stub = stubs.BookmarkManagerStub() - objreg.register('bookmark-manager', stub) - yield stub - objreg.delete('bookmark-manager') - - -@pytest.fixture -def web_history_stub(stubs): - """Fixture which provides a fake web-history object.""" - stub = stubs.WebHistoryStub() - objreg.register('web-history', stub) - yield stub - objreg.delete('web-history') - - @pytest.fixture def session_manager_stub(stubs): """Fixture which provides a fake web-history object.""" @@ -482,3 +455,11 @@ def short_tmpdir(): """A short temporary directory for a XDG_RUNTIME_DIR.""" with tempfile.TemporaryDirectory() as tdir: yield py.path.local(tdir) # pylint: disable=no-member + + +@pytest.fixture() +def init_sql(): + """Initialize the SQL module, and shut it down after the test.""" + sql.init() + yield + sql.close() diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index de7c27a77..ff6b78ae2 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -32,12 +32,9 @@ from qutebrowser.misc import sql @pytest.fixture(autouse=True) -def prerequisites(config_stub, fake_save_manager): +def prerequisites(config_stub, fake_save_manager, init_sql): """Make sure everything is ready to initialize a WebHistory.""" config_stub.data = {'general': {'private-browsing': False}} - sql.init() - yield - sql.close() @pytest.fixture() diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 9870bb4db..a41bc6f02 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -30,6 +30,7 @@ from qutebrowser.completion.models import (miscmodels, urlmodel, configmodel, sortfilter) from qutebrowser.browser import history from qutebrowser.config import sections, value +from qutebrowser.misc import sql def _check_completions(model, expected): @@ -45,17 +46,16 @@ def _check_completions(model, expected): """ assert model.rowCount() == len(expected) for i in range(0, model.rowCount()): - actual_cat = model.item(i) - catname = actual_cat.text() + catidx = model.index(i, 0) + catname = model.data(catidx) assert catname in expected expected_cat = expected[catname] - assert actual_cat.rowCount() == len(expected_cat) - for j in range(0, actual_cat.rowCount()): - name = actual_cat.child(j, 0) - desc = actual_cat.child(j, 1) - misc = actual_cat.child(j, 2) - actual_item = (name.text(), desc.text(), misc.text()) - assert actual_item in expected_cat + assert model.rowCount(catidx) == len(expected_cat) + for j in range(model.rowCount(catidx)): + name = model.data(model.index(j, 0, parent=catidx)) + desc = model.data(model.index(j, 1, parent=catidx)) + misc = model.data(model.index(j, 2, parent=catidx)) + assert (name, desc, misc) in expected_cat # sanity-check the column_widths assert len(model.column_widths) == 3 assert sum(model.column_widths) == 100 @@ -133,42 +133,34 @@ def _mock_view_index(model, category_idx, child_idx, qtbot): @pytest.fixture -def quickmarks(quickmark_manager_stub): - """Pre-populate the quickmark-manager stub with some quickmarks.""" - quickmark_manager_stub.marks = collections.OrderedDict([ - ('aw', 'https://wiki.archlinux.org'), - ('ddg', 'https://duckduckgo.com'), - ('wiki', 'https://wikipedia.org'), - ]) - return quickmark_manager_stub +def quickmarks(init_sql): + """Pre-populate the quickmark database.""" + table = sql.SqlTable('Quickmarks', ['name', 'url'], primary_key='name') + table.insert('aw', 'https://wiki.archlinux.org') + table.insert('ddg', 'https://duckduckgo.com') + table.insert('wiki', 'https://wikipedia.org') @pytest.fixture -def bookmarks(bookmark_manager_stub): - """Pre-populate the bookmark-manager stub with some quickmarks.""" - bookmark_manager_stub.marks = collections.OrderedDict([ - ('https://github.com', 'GitHub'), - ('https://python.org', 'Welcome to Python.org'), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser'), - ]) - return bookmark_manager_stub +def bookmarks(init_sql): + """Pre-populate the bookmark database.""" + table = sql.SqlTable('Bookmarks', ['url', 'title'], primary_key='url') + table.insert('https://github.com', 'GitHub') + table.insert('https://python.org', 'Welcome to Python.org') + table.insert('http://qutebrowser.org', 'qutebrowser | qutebrowser') @pytest.fixture -def web_history(stubs, web_history_stub): - """Pre-populate the web-history stub with some history entries.""" - web_history_stub.history_dict = collections.OrderedDict([ - ('http://qutebrowser.org', history.Entry( - datetime(2015, 9, 5).timestamp(), - QUrl('http://qutebrowser.org'), 'qutebrowser | qutebrowser')), - ('https://python.org', history.Entry( - datetime(2016, 3, 8).timestamp(), - QUrl('https://python.org'), 'Welcome to Python.org')), - ('https://github.com', history.Entry( - datetime(2016, 5, 1).timestamp(), - QUrl('https://github.com'), 'GitHub')), - ]) - return web_history_stub +def web_history(stubs, init_sql): + """Pre-populate the web-history database.""" + table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect'], + primary_key='url') + table.insert('http://qutebrowser.org', 'qutebrowser', + datetime(2015, 9, 5).timestamp(), False) + table.insert('https://python.org', 'Welcome to Python.org', + datetime(2016, 3, 8).timestamp(), False) + table.insert('https://github.com', 'https://github.com', + datetime(2016, 5, 1).timestamp(), False) def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, @@ -237,6 +229,7 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): }) +@pytest.mark.skip def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() @@ -252,6 +245,7 @@ def test_quickmark_completion(qtmodeltester, quickmarks): }) +@pytest.mark.skip def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() @@ -273,33 +267,52 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, Verify that: - quickmarks, bookmarks, and urls are included - - no more than 'web-history-max-items' history entries are included + - no more than 'web-history-max-items' items are included (TODO) - the most recent entries are included """ - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - 'web-history-max-items': 2} + # TODO: time formatting and item limiting + #config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', + # 'web-history-max-items': 2} model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { + # TODO: rearrange columns so address comes first + #"Quickmarks": [ + # ('https://wiki.archlinux.org', 'aw', None), + # ('https://duckduckgo.com', 'ddg', None), + # ('https://wikipedia.org', 'wiki', None), + #], "Quickmarks": [ - ('https://wiki.archlinux.org', 'aw', ''), - ('https://duckduckgo.com', 'ddg', ''), - ('https://wikipedia.org', 'wiki', ''), + ('aw', 'https://wiki.archlinux.org', None), + ('ddg', 'https://duckduckgo.com', None), + ('wiki', 'https://wikipedia.org', None), ], "Bookmarks": [ - ('https://github.com', 'GitHub', ''), - ('https://python.org', 'Welcome to Python.org', ''), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', ''), + ('https://github.com', 'GitHub', None), + ('https://python.org', 'Welcome to Python.org', None), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ], + # TODO: time formatting and item limiting + #"History": [ + # ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + # ('https://github.com', 'GitHub', '2016-05-01'), + # ('http://qutebrowser.org', 'qutebrowser | qutebrowser', + # '2015-09-05', False) + #], "History": [ - ('https://python.org', 'Welcome to Python.org', '2016-03-08'), - ('https://github.com', 'GitHub', '2016-05-01'), + ('http://qutebrowser.org', 'qutebrowser', + datetime(2015, 9, 5).timestamp()), + ('https://python.org', 'Welcome to Python.org', + datetime(2016, 3, 8).timestamp()), + ('https://github.com', 'https://github.com', + datetime(2016, 5, 1).timestamp()), ], }) +@pytest.mark.skip def test_url_completion_delete_bookmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): @@ -318,6 +331,7 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, assert 'http://qutebrowser.org' in bookmarks.marks +@pytest.mark.skip def test_url_completion_delete_quickmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 6b8f01f4d..49cb216dd 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -26,11 +26,7 @@ from qutebrowser.misc import sql from qutebrowser.completion.models import sqlmodel -@pytest.fixture(autouse=True) -def init(): - sql.init() - yield - sql.close() +pytestmark = pytest.mark.usefixtures('init_sql') def _check_model(model, expected): diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 175f2f97e..d11ed7903 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -23,11 +23,7 @@ import pytest from qutebrowser.misc import sql -@pytest.fixture(autouse=True) -def init(): - sql.init() - yield - sql.close() +pytestmark = pytest.mark.usefixtures('init_sql') def test_init(): From c3155afc21137eb074f25cc032d86a8cc731e8bb Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 08:53:50 -0500 Subject: [PATCH 013/337] Rethrow KeyError as DoesNotExistError in urlmarks. From @TheCompiler: To expand on this: I think it's fine to use KeyError on a lower level, i.e. with the SqlTable object with a dict-like interface. However, on this higher level, I think it makes sense to re-raise them as more specific exceptions. --- qutebrowser/browser/commands.py | 2 +- qutebrowser/browser/urlmarks.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 9d19914f8..e5ab0af21 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1261,7 +1261,7 @@ class CommandDispatcher: url = self._current_url() try: quickmark_manager.delete_by_qurl(url) - except KeyError as e: + except urlmarks.DoesNotExistError as e: raise cmdexc.CommandError(str(e)) else: try: diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index db5e14153..ca71b3f0d 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -53,6 +53,13 @@ class InvalidUrlError(Error): pass +class DoesNotExistError(Error): + + """Exception emitted when a given URL does not exist.""" + + pass + + class AlreadyExistsError(Error): """Exception emitted when a given URL does already exist.""" @@ -179,12 +186,14 @@ class QuickmarkManager(UrlMarkManager): try: self.delete(urlstr, field='url') except KeyError: - raise KeyError("Quickmark for '{}' not found!".format(urlstr)) + raise DoesNotExistError("Quickmark for '{}' not found!" + .format(urlstr)) def get(self, name): """Get the URL of the quickmark named name as a QUrl.""" if name not in self: - raise KeyError("Quickmark '{}' does not exist!".format(name)) + raise DoesNotExistError("Quickmark '{}' does not exist!" + .format(name)) urlstr = self[name] try: url = urlutils.fuzzy_url(urlstr, do_search=False) From b381148e06c7e62647c2b8e0ecd7b09e590bf471 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 18:03:00 -0500 Subject: [PATCH 014/337] Unittest CompletionView.completion_item_del. There were no unit tests for this and the various ways it can fail, and I'm about to screw with it a bit. --- .../unit/completion/test_completionwidget.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 88f4aed20..2c8d96713 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -26,6 +26,7 @@ from PyQt5.QtGui import QStandardItem, QColor from qutebrowser.completion import completionwidget from qutebrowser.completion.models import base, sortfilter +from qutebrowser.commands import cmdexc @pytest.fixture @@ -69,6 +70,16 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry, return view +@pytest.fixture +def simplemodel(completionview): + """A filter model wrapped around a completion model with one item.""" + model = base.CompletionModel() + cat = QStandardItem() + cat.appendRow(QStandardItem('foo')) + model.appendRow(cat) + return sortfilter.CompletionFilterModel(model, parent=completionview) + + def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" model = base.CompletionModel() @@ -218,3 +229,29 @@ def test_completion_show(show, rows, quick_complete, completionview, completionview.set_model(None) completionview.completion_item_focus('next') assert not completionview.isVisible() + + +def test_completion_item_del(completionview, simplemodel): + """Test that completion_item_del invokes delete_cur_item in the model.""" + simplemodel.srcmodel.delete_cur_item = unittest.mock.Mock() + completionview.set_model(simplemodel) + completionview.completion_item_focus('next') + completionview.completion_item_del() + assert simplemodel.srcmodel.delete_cur_item.called + + +def test_completion_item_del_no_selection(completionview, simplemodel): + """Test that completion_item_del with no selected index.""" + simplemodel.srcmodel.delete_cur_item = unittest.mock.Mock() + completionview.set_model(simplemodel) + with pytest.raises(cmdexc.CommandError): + completionview.completion_item_del() + assert not simplemodel.srcmodel.delete_cur_item.called + + +def test_completion_item_del_no_func(completionview, simplemodel): + """Test completion_item_del with no delete_cur_item in the model.""" + completionview.set_model(simplemodel) + completionview.completion_item_focus('next') + with pytest.raises(cmdexc.CommandError): + completionview.completion_item_del() From 5bd047b70b2e2f48d5191ef76505bd3154cbed83 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 18:03:49 -0500 Subject: [PATCH 015/337] Fix CompletionView.completion_item_del for new API. The new function based completion models work a little differently so the view needed slightly different error handling. --- qutebrowser/completion/completionwidget.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index f12027340..3d2aef27e 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -368,7 +368,7 @@ class CompletionView(QTreeView): """Delete the current completion item.""" if not self.currentIndex().isValid(): raise cmdexc.CommandError("No item selected!") - try: - self.model().srcmodel.delete_cur_item(self) - except NotImplementedError: + if self.model().srcmodel.delete_cur_item is None: raise cmdexc.CommandError("Cannot delete this item.") + else: + self.model().srcmodel.delete_cur_item(self) From 839d49a8acb544bf4952454b03c028dfbe44409e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 18:14:46 -0500 Subject: [PATCH 016/337] Fix up pylint/flake8 for completion revamp. --- qutebrowser/browser/history.py | 3 +-- qutebrowser/browser/urlmarks.py | 3 +-- qutebrowser/completion/models/base.py | 2 -- qutebrowser/completion/models/miscmodels.py | 2 -- qutebrowser/completion/models/sortfilter.py | 7 ++----- qutebrowser/completion/models/sqlmodel.py | 4 ++-- qutebrowser/completion/models/urlmodel.py | 1 + qutebrowser/config/parsers/keyconf.py | 2 +- tests/unit/browser/webkit/test_history.py | 1 - tests/unit/completion/test_models.py | 20 +++++++++----------- tests/unit/completion/test_sortfilter.py | 2 -- tests/unit/misc/test_sql.py | 2 +- 12 files changed, 18 insertions(+), 31 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index a1fc7dfa2..955de21f2 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -20,9 +20,8 @@ """Simple history which gets written to disk.""" import time -import collections -from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject +from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl from qutebrowser.commands import cmdutils from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils, diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index ca71b3f0d..6d692da0a 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -29,9 +29,8 @@ import os import html import os.path import functools -import collections -from PyQt5.QtCore import QUrl, QObject +from PyQt5.QtCore import QUrl from qutebrowser.utils import (message, usertypes, qtutils, urlutils, standarddir, objreg, log) diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py index 5e2c78c5a..112c6ed00 100644 --- a/qutebrowser/completion/models/base.py +++ b/qutebrowser/completion/models/base.py @@ -26,8 +26,6 @@ Module attributes: from PyQt5.QtCore import Qt from PyQt5.QtGui import QStandardItemModel, QStandardItem -from qutebrowser.utils import usertypes - class CompletionModel(QStandardItemModel): diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index cb87f9d7e..5cb955216 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -19,8 +19,6 @@ """Functions that return miscellaneous completion models.""" -from PyQt5.QtCore import Qt - from qutebrowser.config import config, configdata from qutebrowser.utils import objreg, log, qtutils from qutebrowser.commands import cmdutils diff --git a/qutebrowser/completion/models/sortfilter.py b/qutebrowser/completion/models/sortfilter.py index e5d42607c..92d9aeaf1 100644 --- a/qutebrowser/completion/models/sortfilter.py +++ b/qutebrowser/completion/models/sortfilter.py @@ -25,10 +25,9 @@ Contains: import re -from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex, Qt +from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex from qutebrowser.utils import log, qtutils, debug -from qutebrowser.completion.models import base as completion class CompletionFilterModel(QSortFilterProxyModel): @@ -48,8 +47,6 @@ class CompletionFilterModel(QSortFilterProxyModel): self.srcmodel = source self.pattern = '' self.pattern_re = None - self.lessThan = self.intelligentLessThan - #self._sort_order = self.srcmodel.sort_order or Qt.AscendingOrder def set_pattern(self, val): """Setter for pattern. @@ -137,7 +134,7 @@ class CompletionFilterModel(QSortFilterProxyModel): return True return False - def intelligentLessThan(self, lindex, rindex): + def lessThan(self, lindex, rindex): """Custom sorting implementation. Prefers all items which start with self.pattern. Other than that, uses diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index a5a9bc99a..2f0a5059b 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -22,9 +22,9 @@ import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel -from PyQt5.QtSql import QSqlTableModel, QSqlDatabase, QSqlQuery +from PyQt5.QtSql import QSqlTableModel, QSqlDatabase -from qutebrowser.utils import usertypes, log +from qutebrowser.utils import log class SqlCompletionModel(QAbstractItemModel): diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index bd3efd469..1390fff21 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -21,6 +21,7 @@ from qutebrowser.completion.models import sqlmodel + def url(): """A model which combines bookmarks, quickmarks and web history URLs. diff --git a/qutebrowser/config/parsers/keyconf.py b/qutebrowser/config/parsers/keyconf.py index e4adb8676..5a780a786 100644 --- a/qutebrowser/config/parsers/keyconf.py +++ b/qutebrowser/config/parsers/keyconf.py @@ -27,7 +27,7 @@ from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.config import configdata, textwrapper from qutebrowser.commands import cmdutils, cmdexc -from qutebrowser.utils import log, utils, qtutils, message +from qutebrowser.utils import log, utils, qtutils, message, usertypes from qutebrowser.completion.models import miscmodels diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index ff6b78ae2..844c20fb1 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -28,7 +28,6 @@ from PyQt5.QtCore import QUrl from qutebrowser.browser import history from qutebrowser.utils import objreg, urlutils, usertypes -from qutebrowser.misc import sql @pytest.fixture(autouse=True) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index a41bc6f02..2e687a344 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -26,9 +26,8 @@ import pytest from PyQt5.QtCore import QUrl from PyQt5.QtWidgets import QTreeView -from qutebrowser.completion.models import (miscmodels, urlmodel, configmodel, - sortfilter) from qutebrowser.browser import history +from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value from qutebrowser.misc import sql @@ -544,14 +543,13 @@ def test_url_completion_benchmark(benchmark, config_stub, for e in entries[0:1000]) def bench(): - model = urlmodel.UrlCompletionModel() - filtermodel = sortfilter.CompletionFilterModel(model) - filtermodel.set_pattern('') - filtermodel.set_pattern('e') - filtermodel.set_pattern('ex') - filtermodel.set_pattern('ex ') - filtermodel.set_pattern('ex 1') - filtermodel.set_pattern('ex 12') - filtermodel.set_pattern('ex 123') + model = urlmodel.url() + model.set_pattern('') + model.set_pattern('e') + model.set_pattern('ex') + model.set_pattern('ex ') + model.set_pattern('ex 1') + model.set_pattern('ex 12') + model.set_pattern('ex 123') benchmark(bench) diff --git a/tests/unit/completion/test_sortfilter.py b/tests/unit/completion/test_sortfilter.py index 7d84e9489..92d644810 100644 --- a/tests/unit/completion/test_sortfilter.py +++ b/tests/unit/completion/test_sortfilter.py @@ -21,8 +21,6 @@ import pytest -from PyQt5.QtCore import Qt - from qutebrowser.completion.models import base, sortfilter diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index d11ed7903..889e28a19 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -74,7 +74,7 @@ def test_delete(qtbot): assert list(table) == [('one', 1, False), ('nine', 9, False)] with qtbot.waitSignal(table.changed): table.delete(False, field='lucky') - assert not list(table) == [('thirteen', 13, True)] + assert not list(table) def test_len(): From 52d7d1df0c9ff754dcf7e1214fe7de6f8fd4271e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 12 Feb 2017 18:24:09 -0500 Subject: [PATCH 017/337] Use SQL completer for quickmarks/bookmarks. --- qutebrowser/completion/models/miscmodels.py | 14 +++++--------- tests/unit/completion/test_models.py | 14 ++++++-------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 5cb955216..8bb234521 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -22,7 +22,7 @@ from qutebrowser.config import config, configdata from qutebrowser.utils import objreg, log, qtutils from qutebrowser.commands import cmdutils -from qutebrowser.completion.models import base +from qutebrowser.completion.models import base, sqlmodel def command(): @@ -64,20 +64,16 @@ def helptopic(): def quickmark(): """A CompletionModel filled with all quickmarks.""" model = base.CompletionModel() - cat = model.new_category("Quickmarks") - quickmarks = objreg.get('quickmark-manager').marks.items() - for qm_name, qm_url in quickmarks: - model.new_item(cat, qm_name, qm_url) + model = sqlmodel.SqlCompletionModel(column_widths=(30, 70, 0)) + model.new_category('Quickmarks') return model def bookmark(): """A CompletionModel filled with all bookmarks.""" model = base.CompletionModel() - cat = model.new_category("Bookmarks") - bookmarks = objreg.get('bookmark-manager').marks.items() - for bm_url, bm_title in bookmarks: - model.new_item(cat, bm_url, bm_title) + model = sqlmodel.SqlCompletionModel(column_widths=(30, 70, 0)) + model.new_category('Bookmarks') return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 2e687a344..e1366325c 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -228,7 +228,6 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): }) -@pytest.mark.skip def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() @@ -237,14 +236,13 @@ def test_quickmark_completion(qtmodeltester, quickmarks): _check_completions(model, { "Quickmarks": [ - ('aw', 'https://wiki.archlinux.org', ''), - ('ddg', 'https://duckduckgo.com', ''), - ('wiki', 'https://wikipedia.org', ''), + ('aw', 'https://wiki.archlinux.org', None), + ('ddg', 'https://duckduckgo.com', None), + ('wiki', 'https://wikipedia.org', None), ] }) -@pytest.mark.skip def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() @@ -253,9 +251,9 @@ def test_bookmark_completion(qtmodeltester, bookmarks): _check_completions(model, { "Bookmarks": [ - ('https://github.com', 'GitHub', ''), - ('https://python.org', 'Welcome to Python.org', ''), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', ''), + ('https://github.com', 'GitHub', None), + ('https://python.org', 'Welcome to Python.org', None), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ] }) From b70d5ba901de09aee744993913a58e910d884936 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 08:46:31 -0500 Subject: [PATCH 018/337] Use QSqlQueryModel instead of QSqlTableModel. This allows setting the query as a QSqlQuery instead of a string, which allows: - Escaping quotes - Using LIMIT (needed for history-max-items) - Using ORDER BY (needed for sorting history) - SELECTing columns (needed for quickmark completion) - Creating a custom select (needed for history timestamp formatting) --- qutebrowser/completion/models/sqlmodel.py | 58 ++++++++++++++++------- qutebrowser/misc/sql.py | 18 +++---- tests/unit/completion/test_sqlmodel.py | 15 +++++- 3 files changed, 62 insertions(+), 29 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 2f0a5059b..40b2e27ca 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -22,9 +22,39 @@ import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel -from PyQt5.QtSql import QSqlTableModel, QSqlDatabase +from PyQt5.QtSql import QSqlQuery, QSqlQueryModel, QSqlDatabase from qutebrowser.utils import log +from qutebrowser.misc import sql + + +class SqlCompletionCategory(QSqlQueryModel): + def __init__(self, name, sort_by, sort_order, limit, columns_to_filter, + parent=None): + super().__init__(parent=parent) + self.tablename = name + + query = sql.run_query('select * from {} limit 1'.format(name)) + self._fields = [query.record().fieldName(i) for i in columns_to_filter] + + querystr = 'select * from {} where '.format(self.tablename) + querystr += ' or '.join('{} like ?'.format(f) for f in self._fields) + querystr += " escape '\\'" + + if sort_by: + sortstr = 'asc' if sort_order == Qt.AscendingOrder else 'desc' + querystr += ' order by {} {}'.format(sort_by, sortstr) + + if limit: + querystr += ' limit {}'.format(limit) + + self._querystr = querystr + self.set_pattern('%') + + def set_pattern(self, pattern): + # TODO: kill star-args for run_query + query = sql.run_query(self._querystr, *[pattern for _ in self._fields]) + self.setQuery(query) class SqlCompletionModel(QAbstractItemModel): @@ -58,7 +88,7 @@ class SqlCompletionModel(QAbstractItemModel): self.srcmodel = self # TODO: dummy for compat with old API self.pattern = '' - def new_category(self, name, sort_by=None, sort_order=Qt.AscendingOrder): + def new_category(self, name, sort_by=None, sort_order=None, limit=None): """Create a new completion category and add it to this model. Args: @@ -68,14 +98,10 @@ class SqlCompletionModel(QAbstractItemModel): Return: A new CompletionCategory. """ - database = QSqlDatabase.database() - cat = QSqlTableModel(parent=self, db=database) - cat.setTable(name) - if sort_by: - cat.setSort(cat.fieldIndex(sort_by), sort_order) - cat.select() + cat = SqlCompletionCategory(name, parent=self, sort_by=sort_by, + sort_order=sort_order, limit=limit, + columns_to_filter=self.columns_to_filter) self._categories.append(cat) - return cat def delete_cur_item(self, completion): """Delete the selected item.""" @@ -95,7 +121,7 @@ class SqlCompletionModel(QAbstractItemModel): return if not index.parent().isValid(): if index.column() == 0: - return self._categories[index.row()].tableName() + return self._categories[index.row()].tablename else: table = self._categories[index.parent().row()] idx = table.index(index.row(), index.column()) @@ -177,6 +203,7 @@ class SqlCompletionModel(QAbstractItemModel): Args: pattern: The filter pattern to set. """ + log.completion.debug("Setting completion pattern '{}'".format(pattern)) # TODO: should pattern be saved in the view layer instead? self.pattern = pattern # escape to treat a user input % or _ as a literal, not a wildcard @@ -184,14 +211,9 @@ class SqlCompletionModel(QAbstractItemModel): pattern = pattern.replace('_', '\\_') # treat spaces as wildcards to match any of the typed words pattern = re.sub(r' +', '%', pattern) - for t in self._categories: - fields = (t.record().fieldName(i) for i in self.columns_to_filter) - query = ' or '.join("{} like '%{}%' escape '\\'" - .format(field, pattern) - for field in fields) - log.completion.debug("Setting filter = '{}' for table '{}'" - .format(query, t.tableName())) - t.setFilter(query) + pattern = '%{}%'.format(pattern) + for cat in self._categories: + cat.set_pattern(pattern) def first_item(self): """Return the index of the first child (non-category) in the model.""" diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index fd7257bdb..9065caa48 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -39,7 +39,7 @@ def close(): QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) -def _run_query(querystr, *values): +def run_query(querystr, *values): """Run the given SQL query string on the database. Args: @@ -85,12 +85,12 @@ class SqlTable(QObject): super().__init__(parent) self._name = name self._primary_key = primary_key - _run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))" + run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))" .format(name, ','.join(fields), primary_key)) def __iter__(self): """Iterate rows in the table.""" - result = _run_query("SELECT * FROM {}".format(self._name)) + result = run_query("SELECT * FROM {}".format(self._name)) while result.next(): rec = result.record() yield tuple(rec.value(i) for i in range(rec.count())) @@ -101,13 +101,13 @@ class SqlTable(QObject): Args: key: Primary key value to search for. """ - query = _run_query("SELECT * FROM {} where {} = ?" + query = run_query("SELECT * FROM {} where {} = ?" .format(self._name, self._primary_key), key) return query.next() def __len__(self): """Return the count of rows in the table.""" - result = _run_query("SELECT count(*) FROM {}".format(self._name)) + result = run_query("SELECT count(*) FROM {}".format(self._name)) result.next() return result.value(0) @@ -117,7 +117,7 @@ class SqlTable(QObject): Args: key: Primary key value to fetch. """ - result = _run_query("SELECT * FROM {} where {} = ?" + result = run_query("SELECT * FROM {} where {} = ?" .format(self._name, self._primary_key), key) result.next() rec = result.record() @@ -134,7 +134,7 @@ class SqlTable(QObject): The number of rows deleted. """ field = field or self._primary_key - query = _run_query("DELETE FROM {} where {} = ?" + query = run_query("DELETE FROM {} where {} = ?" .format(self._name, field), value) if not query.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) @@ -149,13 +149,13 @@ class SqlTable(QObject): """ cmd = "REPLACE" if replace else "INSERT" paramstr = ','.join(['?'] * len(values)) - _run_query("{} INTO {} values({})" + run_query("{} INTO {} values({})" .format(cmd, self._name, paramstr), *values) self.changed.emit() def delete_all(self): """Remove all row from the table.""" - _run_query("DELETE FROM {}".format(self._name)) + run_query("DELETE FROM {}".format(self._name)) self.changed.emit() diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 49cb216dd..024b69ec2 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -155,16 +155,19 @@ def test_sorting(sort_by, sort_order, data, expected): ('_', [0], [('A', [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')])], [('A', [('a_b', '', ''), ('__a', '', '')])]), + + ("can't", [0], + [('A', [("can't touch this", '', ''), ('a', '', '')])], + [('A', [("can't touch this", '', '')])]), ]) def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" - model = sqlmodel.SqlCompletionModel() + model = sqlmodel.SqlCompletionModel(columns_to_filter=filter_cols) for name, rows in before: table = sql.SqlTable(name, ['a', 'b', 'c'], primary_key='a') for row in rows: table.insert(*row) model.new_category(name) - model.columns_to_filter = filter_cols model.set_pattern(pattern) _check_model(model, after) @@ -198,3 +201,11 @@ def test_first_last_item(data, first, last): model.new_category(name) assert model.data(model.first_item()) == first assert model.data(model.last_item()) == last + +def test_limit(): + table = sql.SqlTable('test_limit', ['a'], primary_key='a') + for i in range(5): + table.insert([i]) + model = sqlmodel.SqlCompletionModel() + model.new_category('test_limit', limit=3) + assert model.count() == 3 From df995c02a38b3542dc528d9679d9557188c81f01 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 08:59:01 -0500 Subject: [PATCH 019/337] Get rid of varargs in sql.run_query. Things are clearer when just passing a list. --- qutebrowser/browser/history.py | 4 +- qutebrowser/browser/urlmarks.py | 8 ++-- qutebrowser/completion/models/sqlmodel.py | 3 +- qutebrowser/misc/sql.py | 18 ++++----- tests/unit/completion/test_models.py | 24 ++++++------ tests/unit/completion/test_sqlmodel.py | 8 ++-- tests/unit/misc/test_sql.py | 46 +++++++++++------------ 7 files changed, 55 insertions(+), 56 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 955de21f2..d2adea771 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -188,8 +188,8 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" - self.insert(entry.url_str(), entry.title, entry.atime, entry.redirect, - replace=True) + self.insert([entry.url_str(), entry.title, entry.atime, + entry.redirect], replace=True) def get_recent(self): """Get the most recent history entries.""" diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index 6d692da0a..e28966b24 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -128,7 +128,7 @@ class QuickmarkManager(UrlMarkManager): except ValueError: message.error("Invalid quickmark '{}'".format(line)) else: - self.insert(key, url) + self.insert([key, url]) def prompt_save(self, url): """Prompt for a new quickmark name to be added and add it. @@ -168,7 +168,7 @@ class QuickmarkManager(UrlMarkManager): def set_mark(): """Really set the quickmark.""" - self.insert(name, url) + self.insert([name, url]) log.misc.debug("Added quickmark {} for {}".format(name, url)) if name in self: @@ -231,7 +231,7 @@ class BookmarkManager(UrlMarkManager): parts = line.split(maxsplit=1) urlstr = parts[0] title = parts[1] if len(parts) == 2 else '' - self.insert(urlstr, title) + self.insert([urlstr, title]) def add(self, url, title, *, toggle=False): """Add a new bookmark. @@ -259,5 +259,5 @@ class BookmarkManager(UrlMarkManager): else: raise AlreadyExistsError("Bookmark already exists!") else: - self.insert(urlstr, title) + self.insert([urlstr, title]) return True diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 40b2e27ca..b45990e6d 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -52,8 +52,7 @@ class SqlCompletionCategory(QSqlQueryModel): self.set_pattern('%') def set_pattern(self, pattern): - # TODO: kill star-args for run_query - query = sql.run_query(self._querystr, *[pattern for _ in self._fields]) + query = sql.run_query(self._querystr, [pattern for _ in self._fields]) self.setQuery(query) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 9065caa48..11d64bf72 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -39,17 +39,17 @@ def close(): QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) -def run_query(querystr, *values): +def run_query(querystr, values=None): """Run the given SQL query string on the database. Args: - values: positional parameter bindings. + values: A list of positional parameter bindings. """ log.completion.debug('Running SQL query: "{}"'.format(querystr)) database = QSqlDatabase.database() query = QSqlQuery(database) query.prepare(querystr) - for val in values: + for val in values or []: query.addBindValue(val) log.completion.debug('Query bindings: {}'.format(query.boundValues())) if not query.exec_(): @@ -102,7 +102,7 @@ class SqlTable(QObject): key: Primary key value to search for. """ query = run_query("SELECT * FROM {} where {} = ?" - .format(self._name, self._primary_key), key) + .format(self._name, self._primary_key), [key]) return query.next() def __len__(self): @@ -118,7 +118,7 @@ class SqlTable(QObject): key: Primary key value to fetch. """ result = run_query("SELECT * FROM {} where {} = ?" - .format(self._name, self._primary_key), key) + .format(self._name, self._primary_key), [key]) result.next() rec = result.record() return tuple(rec.value(i) for i in range(rec.count())) @@ -135,22 +135,22 @@ class SqlTable(QObject): """ field = field or self._primary_key query = run_query("DELETE FROM {} where {} = ?" - .format(self._name, field), value) + .format(self._name, field), [value]) if not query.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, *values, replace=False): + def insert(self, values, replace=False): """Append a row to the table. Args: - values: Values in the order fields were given on table creation. + values: A list of values to insert. replace: If true, allow inserting over an existing primary key. """ cmd = "REPLACE" if replace else "INSERT" paramstr = ','.join(['?'] * len(values)) run_query("{} INTO {} values({})" - .format(cmd, self._name, paramstr), *values) + .format(cmd, self._name, paramstr), values) self.changed.emit() def delete_all(self): diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index e1366325c..a89041713 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -135,18 +135,18 @@ def _mock_view_index(model, category_idx, child_idx, qtbot): def quickmarks(init_sql): """Pre-populate the quickmark database.""" table = sql.SqlTable('Quickmarks', ['name', 'url'], primary_key='name') - table.insert('aw', 'https://wiki.archlinux.org') - table.insert('ddg', 'https://duckduckgo.com') - table.insert('wiki', 'https://wikipedia.org') + table.insert(['aw', 'https://wiki.archlinux.org']) + table.insert(['ddg', 'https://duckduckgo.com']) + table.insert(['wiki', 'https://wikipedia.org']) @pytest.fixture def bookmarks(init_sql): """Pre-populate the bookmark database.""" table = sql.SqlTable('Bookmarks', ['url', 'title'], primary_key='url') - table.insert('https://github.com', 'GitHub') - table.insert('https://python.org', 'Welcome to Python.org') - table.insert('http://qutebrowser.org', 'qutebrowser | qutebrowser') + table.insert(['https://github.com', 'GitHub']) + table.insert(['https://python.org', 'Welcome to Python.org']) + table.insert(['http://qutebrowser.org', 'qutebrowser | qutebrowser']) @pytest.fixture @@ -154,12 +154,12 @@ def web_history(stubs, init_sql): """Pre-populate the web-history database.""" table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect'], primary_key='url') - table.insert('http://qutebrowser.org', 'qutebrowser', - datetime(2015, 9, 5).timestamp(), False) - table.insert('https://python.org', 'Welcome to Python.org', - datetime(2016, 3, 8).timestamp(), False) - table.insert('https://github.com', 'https://github.com', - datetime(2016, 5, 1).timestamp(), False) + table.insert(['http://qutebrowser.org', 'qutebrowser', + datetime(2015, 9, 5).timestamp(), False]) + table.insert(['https://python.org', 'Welcome to Python.org', + datetime(2016, 3, 8).timestamp(), False]) + table.insert(['https://github.com', 'https://github.com', + datetime(2016, 5, 1).timestamp(), False]) def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 024b69ec2..2c158d3d4 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -67,7 +67,7 @@ def test_count(rowcounts, expected): name = 'Foo' + str(i) table = sql.SqlTable(name, ['a'], primary_key='a') for rownum in range(rowcount): - table.insert(rownum) + table.insert([rownum]) model.new_category(name) assert model.count() == expected @@ -104,7 +104,7 @@ def test_count(rowcounts, expected): def test_sorting(sort_by, sort_order, data, expected): table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') for row in data: - table.insert(*row) + table.insert(row) model = sqlmodel.SqlCompletionModel() model.new_category('Foo', sort_by=sort_by, sort_order=sort_order) _check_model(model, [('Foo', expected)]) @@ -166,7 +166,7 @@ def test_set_pattern(pattern, filter_cols, before, after): for name, rows in before: table = sql.SqlTable(name, ['a', 'b', 'c'], primary_key='a') for row in rows: - table.insert(*row) + table.insert(row) model.new_category(name) model.set_pattern(pattern) _check_model(model, after) @@ -197,7 +197,7 @@ def test_first_last_item(data, first, last): for name, rows in data: table = sql.SqlTable(name, ['a'], primary_key='a') for row in rows: - table.insert(row) + table.insert([row]) model.new_category(name) assert model.data(model.first_item()) == first assert model.data(model.last_item()) == last diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 889e28a19..4a5299b48 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -36,19 +36,19 @@ def test_init(): def test_insert(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') with qtbot.waitSignal(table.changed): - table.insert('one', 1, False) + table.insert(['one', 1, False]) with qtbot.waitSignal(table.changed): - table.insert('wan', 1, False) + table.insert(['wan', 1, False]) with pytest.raises(sql.SqlException): # duplicate primary key - table.insert('one', 1, False) + table.insert(['one', 1, False]) def test_iter(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) - table.insert('nine', 9, False) - table.insert('thirteen', 13, True) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) assert list(table) == [('one', 1, False), ('nine', 9, False), ('thirteen', 13, True)] @@ -56,17 +56,17 @@ def test_iter(): def test_replace(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) + table.insert(['one', 1, False]) with qtbot.waitSignal(table.changed): - table.insert('one', 1, True, replace=True) + table.insert(['one', 1, True], replace=True) assert list(table) == [('one', 1, True)] def test_delete(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) - table.insert('nine', 9, False) - table.insert('thirteen', 13, True) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) with pytest.raises(KeyError): table.delete('nope') with qtbot.waitSignal(table.changed): @@ -80,19 +80,19 @@ def test_delete(qtbot): def test_len(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') assert len(table) == 0 - table.insert('one', 1, False) + table.insert(['one', 1, False]) assert len(table) == 1 - table.insert('nine', 9, False) + table.insert(['nine', 9, False]) assert len(table) == 2 - table.insert('thirteen', 13, True) + table.insert(['thirteen', 13, True]) assert len(table) == 3 def test_contains(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) - table.insert('nine', 9, False) - table.insert('thirteen', 13, True) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) assert 'oone' not in table assert 'ninee' not in table assert 1 not in table @@ -104,9 +104,9 @@ def test_contains(): def test_index(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) - table.insert('nine', 9, False) - table.insert('thirteen', 13, True) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) assert table['one'] == ('one', 1, False) assert table['nine'] == ('nine', 9, False) assert table['thirteen'] == ('thirteen', 13, True) @@ -114,9 +114,9 @@ def test_index(): def test_delete_all(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert('one', 1, False) - table.insert('nine', 9, False) - table.insert('thirteen', 13, True) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) with qtbot.waitSignal(table.changed): table.delete_all() assert list(table) == [] From 3f6f03e3250194672f8aa87c8973238d6d382797 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 09:06:42 -0500 Subject: [PATCH 020/337] Respect web-history-max-items with SQL. Use a LIMIT with the sql competion model to limit history entries as the old completion implementation did. --- qutebrowser/completion/models/urlmodel.py | 4 +++- tests/unit/completion/test_models.py | 6 ++---- tests/unit/completion/test_sqlmodel.py | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 1390fff21..b844fa93e 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -20,6 +20,7 @@ """Function to return the url completion model for the `open` command.""" from qutebrowser.completion.models import sqlmodel +from qutebrowser.config import config def url(): @@ -32,7 +33,8 @@ def url(): model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), columns_to_filter=[urlcol, textcol]) - model.new_category('History') + model.new_category('History', + limit=config.get('completion', 'web-history-max-items')) model.new_category('Quickmarks') model.new_category('Bookmarks') return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index a89041713..91ac8286f 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -268,8 +268,8 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, - the most recent entries are included """ # TODO: time formatting and item limiting - #config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - # 'web-history-max-items': 2} + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', + 'web-history-max-items': 2} model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -303,8 +303,6 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, datetime(2015, 9, 5).timestamp()), ('https://python.org', 'Welcome to Python.org', datetime(2016, 3, 8).timestamp()), - ('https://github.com', 'https://github.com', - datetime(2016, 5, 1).timestamp()), ], }) diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 2c158d3d4..1044a831f 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -202,6 +202,7 @@ def test_first_last_item(data, first, last): assert model.data(model.first_item()) == first assert model.data(model.last_item()) == last + def test_limit(): table = sql.SqlTable('test_limit', ['a'], primary_key='a') for i in range(5): From 9f27a9a5d77a5bdd63d9c6a1d1f52e8429d87a97 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 20:32:44 -0500 Subject: [PATCH 021/337] Implement column selectors for sql completion. A SQL completion category can now provide a customized column expression for the select statement. This enables the url model to format timestamps, as well as rearrange the name and url in the quickmark section. --- qutebrowser/completion/models/sqlmodel.py | 12 +++++++---- qutebrowser/completion/models/urlmodel.py | 8 ++++++-- tests/unit/completion/test_models.py | 25 +++++------------------ tests/unit/completion/test_sqlmodel.py | 8 ++++++++ 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index b45990e6d..8a0abda2b 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -29,15 +29,15 @@ from qutebrowser.misc import sql class SqlCompletionCategory(QSqlQueryModel): - def __init__(self, name, sort_by, sort_order, limit, columns_to_filter, - parent=None): + def __init__(self, name, sort_by, sort_order, limit, select, + columns_to_filter, parent=None): super().__init__(parent=parent) self.tablename = name query = sql.run_query('select * from {} limit 1'.format(name)) self._fields = [query.record().fieldName(i) for i in columns_to_filter] - querystr = 'select * from {} where '.format(self.tablename) + querystr = 'select {} from {} where '.format(select, name) querystr += ' or '.join('{} like ?'.format(f) for f in self._fields) querystr += " escape '\\'" @@ -87,18 +87,22 @@ class SqlCompletionModel(QAbstractItemModel): self.srcmodel = self # TODO: dummy for compat with old API self.pattern = '' - def new_category(self, name, sort_by=None, sort_order=None, limit=None): + def new_category(self, name, select='*', sort_by=None, + sort_order=None, limit=None): """Create a new completion category and add it to this model. Args: name: Name of category, and the table in the database. + select: A custom result column expression for the select statement. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Sorting order, if sort_by is non-None. + limit: Maximum row count to return on a query. Return: A new CompletionCategory. """ cat = SqlCompletionCategory(name, parent=self, sort_by=sort_by, sort_order=sort_order, limit=limit, + select=select, columns_to_filter=self.columns_to_filter) self._categories.append(cat) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index b844fa93e..164a5da15 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -33,8 +33,12 @@ def url(): model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), columns_to_filter=[urlcol, textcol]) + limit = config.get('completion', 'web-history-max-items') + timefmt = config.get('completion', 'timestamp-format') + select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) model.new_category('History', - limit=config.get('completion', 'web-history-max-items')) - model.new_category('Quickmarks') + limit=limit, + select='url, title, {}'.format(select_time)) + model.new_category('Quickmarks', select='url, name') model.new_category('Bookmarks') return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 91ac8286f..21b65e092 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -275,34 +275,19 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, qtmodeltester.check(model) _check_completions(model, { - # TODO: rearrange columns so address comes first - #"Quickmarks": [ - # ('https://wiki.archlinux.org', 'aw', None), - # ('https://duckduckgo.com', 'ddg', None), - # ('https://wikipedia.org', 'wiki', None), - #], "Quickmarks": [ - ('aw', 'https://wiki.archlinux.org', None), - ('ddg', 'https://duckduckgo.com', None), - ('wiki', 'https://wikipedia.org', None), + ('https://wiki.archlinux.org', 'aw', None), + ('https://duckduckgo.com', 'ddg', None), + ('https://wikipedia.org', 'wiki', None), ], "Bookmarks": [ ('https://github.com', 'GitHub', None), ('https://python.org', 'Welcome to Python.org', None), ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ], - # TODO: time formatting and item limiting - #"History": [ - # ('https://python.org', 'Welcome to Python.org', '2016-03-08'), - # ('https://github.com', 'GitHub', '2016-05-01'), - # ('http://qutebrowser.org', 'qutebrowser | qutebrowser', - # '2015-09-05', False) - #], "History": [ - ('http://qutebrowser.org', 'qutebrowser', - datetime(2015, 9, 5).timestamp()), - ('https://python.org', 'Welcome to Python.org', - datetime(2016, 3, 8).timestamp()), + ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ('https://python.org', 'Welcome to Python.org', '2016-03-08'), ], }) diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 1044a831f..a0dbbe7b3 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -210,3 +210,11 @@ def test_limit(): model = sqlmodel.SqlCompletionModel() model.new_category('test_limit', limit=3) assert model.count() == 3 + + +def test_select(): + table = sql.SqlTable('test_select', ['a', 'b', 'c'], primary_key='a') + table.insert(['foo', 'bar', 'baz']) + model = sqlmodel.SqlCompletionModel() + model.new_category('test_select', select='b, c, a') + _check_model(model, [('test_select', [('bar', 'baz', 'foo')])]) From fe808787887638ead2dba61c7a7ff16760895833 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 22:10:49 -0500 Subject: [PATCH 022/337] Implement custom where clause in SQL. Allow categories to specify a WHERE clause that applies in addition to the pattern filter. This allows the url completion model to filter out redirect entries. This also fixed the usage of ESCAPE so it applies to all the LIKE statements. --- qutebrowser/completion/models/sqlmodel.py | 16 ++++++++++------ qutebrowser/completion/models/urlmodel.py | 3 ++- tests/unit/completion/test_models.py | 3 +++ tests/unit/completion/test_sqlmodel.py | 13 +++++++++++++ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 8a0abda2b..39182628c 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -29,7 +29,7 @@ from qutebrowser.misc import sql class SqlCompletionCategory(QSqlQueryModel): - def __init__(self, name, sort_by, sort_order, limit, select, + def __init__(self, name, sort_by, sort_order, limit, select, where, columns_to_filter, parent=None): super().__init__(parent=parent) self.tablename = name @@ -37,9 +37,12 @@ class SqlCompletionCategory(QSqlQueryModel): query = sql.run_query('select * from {} limit 1'.format(name)) self._fields = [query.record().fieldName(i) for i in columns_to_filter] - querystr = 'select {} from {} where '.format(select, name) - querystr += ' or '.join('{} like ?'.format(f) for f in self._fields) - querystr += " escape '\\'" + querystr = 'select {} from {} where ('.format(select, name) + querystr += ' or '.join("{} like ? escape '\\'".format(f) + for f in self._fields) + querystr += ')' + if where: + querystr += ' and ' + where if sort_by: sortstr = 'asc' if sort_order == Qt.AscendingOrder else 'desc' @@ -87,13 +90,14 @@ class SqlCompletionModel(QAbstractItemModel): self.srcmodel = self # TODO: dummy for compat with old API self.pattern = '' - def new_category(self, name, select='*', sort_by=None, + def new_category(self, name, select='*', where=None, sort_by=None, sort_order=None, limit=None): """Create a new completion category and add it to this model. Args: name: Name of category, and the table in the database. select: A custom result column expression for the select statement. + where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Sorting order, if sort_by is non-None. limit: Maximum row count to return on a query. @@ -102,7 +106,7 @@ class SqlCompletionModel(QAbstractItemModel): """ cat = SqlCompletionCategory(name, parent=self, sort_by=sort_by, sort_order=sort_order, limit=limit, - select=select, + select=select, where=where, columns_to_filter=self.columns_to_filter) self._categories.append(cat) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 164a5da15..2f573aab7 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -38,7 +38,8 @@ def url(): select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) model.new_category('History', limit=limit, - select='url, title, {}'.format(select_time)) + select='url, title, {}'.format(select_time), + where='not redirect') model.new_category('Quickmarks', select='url, name') model.new_category('Bookmarks') return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 21b65e092..d0732f660 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -154,6 +154,8 @@ def web_history(stubs, init_sql): """Pre-populate the web-history database.""" table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect'], primary_key='url') + table.insert(['http://some-redirect.example.com', 'redirect', + datetime(2016, 9, 5).timestamp(), True]) table.insert(['http://qutebrowser.org', 'qutebrowser', datetime(2015, 9, 5).timestamp(), False]) table.insert(['https://python.org', 'Welcome to Python.org', @@ -266,6 +268,7 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, - quickmarks, bookmarks, and urls are included - no more than 'web-history-max-items' items are included (TODO) - the most recent entries are included + - redirect entries are not included """ # TODO: time formatting and item limiting config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index a0dbbe7b3..b94e8e048 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -156,6 +156,10 @@ def test_sorting(sort_by, sort_order, data, expected): [('A', [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')])], [('A', [('a_b', '', ''), ('__a', '', '')])]), + ('%', [0, 1], + [('A', [('\\foo', '\\bar', '')])], + [('A', [])]), + ("can't", [0], [('A', [("can't touch this", '', ''), ('a', '', '')])], [('A', [("can't touch this", '', '')])]), @@ -218,3 +222,12 @@ def test_select(): model = sqlmodel.SqlCompletionModel() model.new_category('test_select', select='b, c, a') _check_model(model, [('test_select', [('bar', 'baz', 'foo')])]) + + +def test_where(): + table = sql.SqlTable('test_where', ['a', 'b', 'c'], primary_key='a') + table.insert(['foo', 'bar', False]) + table.insert(['baz', 'biz', True]) + model = sqlmodel.SqlCompletionModel() + model.new_category('test_where', where='not c') + _check_model(model, [('test_where', [('foo', 'bar', False)])]) From d89898ef7d83c7bf26d1215e58d172c47e08b39c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 14 Feb 2017 22:43:33 -0500 Subject: [PATCH 023/337] Implement delete_cur_item for sql completion. This re-enables bookmark/quickmark deletion for url completions via the new SQL interface. --- qutebrowser/completion/models/sqlmodel.py | 7 ++-- qutebrowser/completion/models/urlmodel.py | 43 +++++++++++++++++++---- tests/unit/completion/test_models.py | 28 +++++++++------ 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 39182628c..dc66f3480 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -82,13 +82,14 @@ class SqlCompletionModel(QAbstractItemModel): """ def __init__(self, column_widths=(30, 70, 0), columns_to_filter=None, - parent=None): + delete_cur_item=None, parent=None): super().__init__(parent) self.columns_to_filter = columns_to_filter or [0] self.column_widths = column_widths self._categories = [] self.srcmodel = self # TODO: dummy for compat with old API self.pattern = '' + self.delete_cur_item = delete_cur_item def new_category(self, name, select='*', where=None, sort_by=None, sort_order=None, limit=None): @@ -110,10 +111,6 @@ class SqlCompletionModel(QAbstractItemModel): columns_to_filter=self.columns_to_filter) self._categories.append(cat) - def delete_cur_item(self, completion): - """Delete the selected item.""" - raise NotImplementedError - def data(self, index, role=Qt.DisplayRole): """Return the item data for index. diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 2f573aab7..161a9266b 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -21,6 +21,39 @@ from qutebrowser.completion.models import sqlmodel from qutebrowser.config import config +from qutebrowser.utils import qtutils, log, objreg + + +_URLCOL = 0 +_TEXTCOL = 1 + + +def _delete_url(completion): + """Delete the selected item. + + Args: + completion: The Completion object to use. + """ + index = completion.currentIndex() + qtutils.ensure_valid(index) + category = index.parent() + index = category.child(index.row(), _URLCOL) + catname = category.data() + url = index.data() + qtutils.ensure_valid(category) + + if catname == 'Bookmarks': + log.completion.debug('Deleting bookmark {}'.format(url)) + bookmark_manager = objreg.get('bookmark-manager') + bookmark_manager.delete(url) + else: + assert catname == 'Quickmarks', 'Unknown category {}'.format(catname) + quickmark_manager = objreg.get('quickmark-manager') + sibling = index.sibling(index.row(), _TEXTCOL) + qtutils.ensure_valid(sibling) + name = sibling.data() + log.completion.debug('Deleting quickmark {}'.format(name)) + quickmark_manager.delete(name) def url(): @@ -28,18 +61,16 @@ def url(): Used for the `open` command. """ - urlcol = 0 - textcol = 1 - model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), - columns_to_filter=[urlcol, textcol]) + columns_to_filter=[_URLCOL, _TEXTCOL], + delete_cur_item=_delete_url) limit = config.get('completion', 'web-history-max-items') timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) + model.new_category('Quickmarks', select='url, name') + model.new_category('Bookmarks') model.new_category('History', limit=limit, select='url, title, {}'.format(select_time), where='not redirect') - model.new_category('Quickmarks', select='url, name') - model.new_category('Bookmarks') return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index d0732f660..ddc746aa6 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -30,6 +30,7 @@ from qutebrowser.browser import history from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value from qutebrowser.misc import sql +from qutebrowser.utils import objreg def _check_completions(model, expected): @@ -115,7 +116,7 @@ def _patch_config_section_desc(monkeypatch, stubs, symbol): monkeypatch.setattr(symbol, section_desc) -def _mock_view_index(model, category_idx, child_idx, qtbot): +def _mock_view_index(model, category_num, child_num, qtbot): """Create a tree view from a model and set the current index. Args: @@ -126,8 +127,9 @@ def _mock_view_index(model, category_idx, child_idx, qtbot): view = QTreeView() qtbot.add_widget(view) view.setModel(model) - idx = model.indexFromItem(model.item(category_idx).child(child_idx)) - view.setCurrentIndex(idx) + parent = model.index(category_num, 0) + child = model.index(child_num, 0, parent=parent) + view.setCurrentIndex(child) return view @@ -138,6 +140,9 @@ def quickmarks(init_sql): table.insert(['aw', 'https://wiki.archlinux.org']) table.insert(['ddg', 'https://duckduckgo.com']) table.insert(['wiki', 'https://wikipedia.org']) + objreg.register('quickmark-manager', table) + yield table + objreg.delete('quickmark-manager') @pytest.fixture @@ -147,6 +152,9 @@ def bookmarks(init_sql): table.insert(['https://github.com', 'GitHub']) table.insert(['https://python.org', 'Welcome to Python.org']) table.insert(['http://qutebrowser.org', 'qutebrowser | qutebrowser']) + objreg.register('bookmark-manager', table) + yield table + objreg.delete('bookmark-manager') @pytest.fixture @@ -295,7 +303,6 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, }) -@pytest.mark.skip def test_url_completion_delete_bookmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): @@ -309,12 +316,11 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, # delete item (1, 0) -> (bookmarks, 'https://github.com' ) view = _mock_view_index(model, 1, 0, qtbot) model.delete_cur_item(view) - assert 'https://github.com' not in bookmarks.marks - assert 'https://python.org' in bookmarks.marks - assert 'http://qutebrowser.org' in bookmarks.marks + assert 'https://github.com' not in bookmarks + assert 'https://python.org' in bookmarks + assert 'http://qutebrowser.org' in bookmarks -@pytest.mark.skip def test_url_completion_delete_quickmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): @@ -328,9 +334,9 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, # delete item (0, 1) -> (quickmarks, 'ddg' ) view = _mock_view_index(model, 0, 1, qtbot) model.delete_cur_item(view) - assert 'aw' in quickmarks.marks - assert 'ddg' not in quickmarks.marks - assert 'wiki' in quickmarks.marks + assert 'aw' in quickmarks + assert 'ddg' not in quickmarks + assert 'wiki' in quickmarks def test_session_completion(qtmodeltester, session_manager_stub): From 0e650ad719ecc9b635dc53865073c65732b4bceb Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 09:06:23 -0500 Subject: [PATCH 024/337] Return namedtuples from SqlTable. Instead of returning a regular tuple and trying to remember which index maps to which field, return named tuples that allow accessing the fields by name. --- qutebrowser/browser/urlmarks.py | 2 +- qutebrowser/misc/sql.py | 7 +++++-- tests/unit/completion/test_sqlmodel.py | 6 ++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index e28966b24..d9e49b5c2 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -193,7 +193,7 @@ class QuickmarkManager(UrlMarkManager): if name not in self: raise DoesNotExistError("Quickmark '{}' does not exist!" .format(name)) - urlstr = self[name] + urlstr = self[name].url try: url = urlutils.fuzzy_url(urlstr, do_search=False) except urlutils.InvalidUrlError as e: diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 11d64bf72..f23d791ad 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -24,6 +24,8 @@ from PyQt5.QtSql import QSqlDatabase, QSqlQuery from qutebrowser.utils import log +import collections + def init(): """Initialize the SQL database connection.""" @@ -87,13 +89,14 @@ class SqlTable(QObject): self._primary_key = primary_key run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))" .format(name, ','.join(fields), primary_key)) + self.Entry = collections.namedtuple(name + '_Entry', fields) def __iter__(self): """Iterate rows in the table.""" result = run_query("SELECT * FROM {}".format(self._name)) while result.next(): rec = result.record() - yield tuple(rec.value(i) for i in range(rec.count())) + yield self.Entry(*[rec.value(i) for i in range(rec.count())]) def __contains__(self, key): """Return whether the table contains the matching item. @@ -121,7 +124,7 @@ class SqlTable(QObject): .format(self._name, self._primary_key), [key]) result.next() rec = result.record() - return tuple(rec.value(i) for i in range(rec.count())) + return self.Entry(*[rec.value(i) for i in range(rec.count())]) def delete(self, value, field=None): """Remove all rows for which `field` equals `value`. diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index b94e8e048..d3e3ada96 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -231,3 +231,9 @@ def test_where(): model = sqlmodel.SqlCompletionModel() model.new_category('test_where', where='not c') _check_model(model, [('test_where', [('foo', 'bar', False)])]) + +def test_entry(): + table = sql.SqlTable('test_entry', ['a', 'b', 'c'], primary_key='a') + assert hasattr(table.Entry, 'a') + assert hasattr(table.Entry, 'b') + assert hasattr(table.Entry, 'c') From ea9217a61f9b49940dcb2d82fd595b0cb6e16f32 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 09:07:07 -0500 Subject: [PATCH 025/337] Fix qutescheme for new SQL backend. The qute://history and qute://bookmarks handlers were added during my work, and had to be adapted to the SQL-based history backend. --- qutebrowser/browser/qutescheme.py | 2 +- tests/unit/browser/test_qutescheme.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index b0c555e4e..23b009f6d 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -198,7 +198,7 @@ def history_data(start_time): # noqa Arguments: reverse -- whether to reverse the history_dict before iterating. """ - history = objreg.get('web-history').history_dict.values() + history = list(objreg.get('web-history')) if reverse: history = reversed(history) diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 92ad30574..d5e60efab 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -96,7 +96,7 @@ class TestHistoryHandler: return items @pytest.fixture - def fake_web_history(self, fake_save_manager, tmpdir): + def fake_web_history(self, fake_save_manager, tmpdir, init_sql): """Create a fake web-history and register it into objreg.""" web_history = history.WebHistory(tmpdir.dirname, 'fake-history') objreg.register('web-history', web_history) From ffd044b52b1ad07f7f9c62d70e505e5367da9b72 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 12:46:37 -0500 Subject: [PATCH 026/337] Fix pylint and flake8 for SQL work. Vanity commit. This also touches up a few comments. --- qutebrowser/browser/history.py | 2 +- qutebrowser/completion/models/sqlmodel.py | 12 ++++++------ qutebrowser/completion/models/urlmodel.py | 6 +++--- qutebrowser/misc/sql.py | 22 ++++++++++++---------- tests/unit/completion/test_completer.py | 2 +- tests/unit/completion/test_sqlmodel.py | 1 + 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index d2adea771..b6d328942 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -36,7 +36,7 @@ class Entry: Attributes: atime: The time the page was accessed. url: The URL which was accessed as QUrl. - redirect: If True, don't save this entry to disk + redirect: If True, don't show this entry in completion """ def __init__(self, atime, url, title, redirect=False): diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index dc66f3480..624cac80d 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -22,13 +22,13 @@ import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel -from PyQt5.QtSql import QSqlQuery, QSqlQueryModel, QSqlDatabase +from PyQt5.QtSql import QSqlQueryModel from qutebrowser.utils import log from qutebrowser.misc import sql -class SqlCompletionCategory(QSqlQueryModel): +class _SqlCompletionCategory(QSqlQueryModel): def __init__(self, name, sort_by, sort_order, limit, select, where, columns_to_filter, parent=None): super().__init__(parent=parent) @@ -105,10 +105,10 @@ class SqlCompletionModel(QAbstractItemModel): Return: A new CompletionCategory. """ - cat = SqlCompletionCategory(name, parent=self, sort_by=sort_by, - sort_order=sort_order, limit=limit, - select=select, where=where, - columns_to_filter=self.columns_to_filter) + cat = _SqlCompletionCategory(name, parent=self, sort_by=sort_by, + sort_order=sort_order, limit=limit, + select=select, where=where, + columns_to_filter=self.columns_to_filter) self._categories.append(cat) def data(self, index, role=Qt.DisplayRole): diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 161a9266b..387603dd5 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -39,13 +39,13 @@ def _delete_url(completion): category = index.parent() index = category.child(index.row(), _URLCOL) catname = category.data() - url = index.data() qtutils.ensure_valid(category) if catname == 'Bookmarks': - log.completion.debug('Deleting bookmark {}'.format(url)) + urlstr = index.data() + log.completion.debug('Deleting bookmark {}'.format(urlstr)) bookmark_manager = objreg.get('bookmark-manager') - bookmark_manager.delete(url) + bookmark_manager.delete(urlstr) else: assert catname == 'Quickmarks', 'Unknown category {}'.format(catname) quickmark_manager = objreg.get('quickmark-manager') diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index f23d791ad..53b23829f 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -65,6 +65,7 @@ class SqlTable(QObject): """Interface to a sql table. Attributes: + Entry: The class wrapping row data from this table. _name: Name of the SQL table this wraps. _primary_key: The primary key of the table. @@ -87,8 +88,9 @@ class SqlTable(QObject): super().__init__(parent) self._name = name self._primary_key = primary_key - run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))" - .format(name, ','.join(fields), primary_key)) + run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))".format( + name, ','.join(fields), primary_key)) + # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) def __iter__(self): @@ -104,8 +106,8 @@ class SqlTable(QObject): Args: key: Primary key value to search for. """ - query = run_query("SELECT * FROM {} where {} = ?" - .format(self._name, self._primary_key), [key]) + query = run_query("SELECT * FROM {} where {} = ?".format( + self._name, self._primary_key), [key]) return query.next() def __len__(self): @@ -120,8 +122,8 @@ class SqlTable(QObject): Args: key: Primary key value to fetch. """ - result = run_query("SELECT * FROM {} where {} = ?" - .format(self._name, self._primary_key), [key]) + result = run_query("SELECT * FROM {} where {} = ?".format( + self._name, self._primary_key), [key]) result.next() rec = result.record() return self.Entry(*[rec.value(i) for i in range(rec.count())]) @@ -137,8 +139,8 @@ class SqlTable(QObject): The number of rows deleted. """ field = field or self._primary_key - query = run_query("DELETE FROM {} where {} = ?" - .format(self._name, field), [value]) + query = run_query("DELETE FROM {} where {} = ?".format( + self._name, field), [value]) if not query.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() @@ -152,8 +154,8 @@ class SqlTable(QObject): """ cmd = "REPLACE" if replace else "INSERT" paramstr = ','.join(['?'] * len(values)) - run_query("{} INTO {} values({})" - .format(cmd, self._name, paramstr), values) + run_query("{} INTO {} values({})".format(cmd, self._name, paramstr), + values) self.changed.emit() def delete_all(self): diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index 34c9a2d19..dce105c27 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -36,7 +36,7 @@ class FakeCompletionModel(QStandardItemModel): def __init__(self, kind, *pos_args, parent=None): super().__init__(parent) self.kind = kind - self.pos_args = [*pos_args] + self.pos_args = list(pos_args) class CompletionWidgetStub(QObject): diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index d3e3ada96..f6daa722c 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -232,6 +232,7 @@ def test_where(): model.new_category('test_where', where='not c') _check_model(model, [('test_where', [('foo', 'bar', False)])]) + def test_entry(): table = sql.SqlTable('test_entry', ['a', 'b', 'c'], primary_key='a') assert hasattr(table.Entry, 'a') From 02fb1a037ce59027f500ccec2d74e0e843890999 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 20:55:27 -0500 Subject: [PATCH 027/337] Implement canFetchMore for SQL completion. This just forwards canFetchMore and fetchMore to the underlying tables. It seems to be returning True and fetching in some cases (with a large history), so I guess it is useful? --- qutebrowser/completion/models/sqlmodel.py | 31 +++++++++++++++++++++-- qutebrowser/misc/sql.py | 4 +-- qutebrowser/utils/log.py | 1 + 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 624cac80d..fcd8d37c6 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -91,6 +91,19 @@ class SqlCompletionModel(QAbstractItemModel): self.pattern = '' self.delete_cur_item = delete_cur_item + def _cat_from_idx(self, index): + """Return the category pointed to by the given index. + + Args: + idx: A QModelIndex + Returns: + A _SqlCompletionCategory if the index points at one, else None + """ + # items hold an index to the parent category in their internalPointer + # categories have an empty internalPointer + if index.isValid() and not index.internalPointer(): + return self._categories[index.row()] + def new_category(self, name, select='*', where=None, sort_by=None, sort_order=None, limit=None): """Create a new completion category and add it to this model. @@ -184,17 +197,31 @@ class SqlCompletionModel(QAbstractItemModel): if not parent.isValid(): # top-level return len(self._categories) - elif parent.internalPointer() or parent.column() != 0: + cat = self._cat_from_idx(parent) + if not cat or parent.column() != 0: # item or nonzero category column (only first col has children) return 0 else: # category - return self._categories[parent.row()].rowCount() + return cat.rowCount() def columnCount(self, parent=QModelIndex()): # pylint: disable=unused-argument return 3 + def canFetchMore(self, parent): + """Override to forward the call to the tables.""" + cat = self._cat_from_idx(parent) + if cat: + return cat.canFetchMore() + return False + + def fetchMore(self, parent): + """Override to forward the call to the tables.""" + cat = self._cat_from_idx(parent) + if cat: + cat.fetchMore() + def count(self): """Return the count of non-category items.""" return sum(t.rowCount() for t in self._categories) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 53b23829f..c051acf03 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -47,13 +47,13 @@ def run_query(querystr, values=None): Args: values: A list of positional parameter bindings. """ - log.completion.debug('Running SQL query: "{}"'.format(querystr)) + log.sql.debug('Running SQL query: "{}"'.format(querystr)) database = QSqlDatabase.database() query = QSqlQuery(database) query.prepare(querystr) for val in values or []: query.addBindValue(val) - log.completion.debug('Query bindings: {}'.format(query.boundValues())) + log.sql.debug('Query bindings: {}'.format(query.boundValues())) if not query.exec_(): raise SqlException('Failed to exec query "{}": "{}"'.format( querystr, query.lastError().text())) diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py index c2abbfb87..513c05062 100644 --- a/qutebrowser/utils/log.py +++ b/qutebrowser/utils/log.py @@ -141,6 +141,7 @@ sessions = logging.getLogger('sessions') webelem = logging.getLogger('webelem') prompt = logging.getLogger('prompt') network = logging.getLogger('network') +sql = logging.getLogger('sql') ram_handler = None From 6a04c4b3e898b258cd801b906bdd860058bf0f83 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 21:28:03 -0500 Subject: [PATCH 028/337] Allow replacing quickmark with SQL backend. This functionality was lost with the transition to SQL. The user should be able to replace a quickmark if they answer 'yes' to the prompt. --- qutebrowser/browser/urlmarks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index d9e49b5c2..5f7a4fb77 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -168,7 +168,7 @@ class QuickmarkManager(UrlMarkManager): def set_mark(): """Really set the quickmark.""" - self.insert([name, url]) + self.insert([name, url], replace=True) log.misc.debug("Added quickmark {} for {}".format(name, url)) if name in self: From be07107b1cd0d0eff91994d48c2932be811e584c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 15 Feb 2017 21:29:00 -0500 Subject: [PATCH 029/337] Fix end2end completion tests for SQL backend. Change the logging to report the completion function name and have the end2end tests check for this. Remove the tests for realtime completion, as it was decided this is not an important feature and the code is much simpler without it. --- qutebrowser/completion/completer.py | 9 ++--- tests/end2end/features/completion.feature | 40 ++++--------------- tests/end2end/features/test_completion_bdd.py | 2 +- 3 files changed, 12 insertions(+), 39 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 8742156a3..f403354c3 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -97,6 +97,7 @@ class Completer(QObject): if not before_cursor: # '|' or 'set|' model = miscmodels.command() + log.completion.debug('Starting command completion') return sortfilter.CompletionFilterModel(source=model, parent=self) try: cmd = cmdutils.cmd_dict[before_cursor[0]] @@ -113,6 +114,8 @@ class Completer(QObject): if completion is None: return None model = self._get_completion_model(completion, before_cursor[1:]) + log.completion.debug('Starting {} completion' + .format(completion.__name__)) return model def _quote(self, s): @@ -237,11 +240,7 @@ class Completer(QObject): pattern = pattern.strip("'\"") model = self._get_new_completion(before_cursor, pattern) - - log.completion.debug("Setting completion model to {} with pattern '{}'" - .format(model.srcmodel.__class__.__name__ if model else 'None', - pattern)) - + log.completion.debug("Setting pattern to '{}'".format(pattern)) completion.set_model(model, pattern) def _change_completed_part(self, newtext, before, after, immediate=False): diff --git a/tests/end2end/features/completion.feature b/tests/end2end/features/completion.feature index b6c62336c..e93518199 100644 --- a/tests/end2end/features/completion.feature +++ b/tests/end2end/features/completion.feature @@ -32,23 +32,23 @@ Feature: Using completion Scenario: Using command completion When I run :set-cmd-text : - Then the completion model should be CommandCompletionModel + Then the completion model should be command Scenario: Using help completion When I run :set-cmd-text -s :help - Then the completion model should be HelpCompletionModel + Then the completion model should be helptopic Scenario: Using quickmark completion When I run :set-cmd-text -s :quickmark-load - Then the completion model should be QuickmarkCompletionModel + Then the completion model should be quickmark Scenario: Using bookmark completion When I run :set-cmd-text -s :bookmark-load - Then the completion model should be BookmarkCompletionModel + Then the completion model should be bookmark Scenario: Using bind completion When I run :set-cmd-text -s :bind X - Then the completion model should be BindCompletionModel + Then the completion model should be bind Scenario: Using session completion Given I open data/hello.txt @@ -62,37 +62,11 @@ Feature: Using completion Scenario: Using option completion When I run :set-cmd-text -s :set colors - Then the completion model should be SettingOptionCompletionModel + Then the completion model should be option Scenario: Using value completion When I run :set-cmd-text -s :set colors statusbar.bg - Then the completion model should be SettingValueCompletionModel - - Scenario: Updating the completion in realtime - Given I have a fresh instance - And I set completion -> quick-complete to false - When I open data/hello.txt - And I run :set-cmd-text -s :buffer - And I run :completion-item-focus next - And I open data/hello2.txt in a new background tab - And I run :completion-item-focus next - And I open data/hello3.txt in a new background tab - And I run :completion-item-focus next - And I run :command-accept - Then the following tabs should be open: - - data/hello.txt - - data/hello2.txt - - data/hello3.txt (active) - - Scenario: Updating the value completion in realtime - Given I set colors -> statusbar.bg to green - When I run :set-cmd-text -s :set colors statusbar.bg - And I set colors -> statusbar.bg to yellow - And I run :completion-item-focus next - And I run :completion-item-focus next - And I set colors -> statusbar.bg to red - And I run :command-accept - Then colors -> statusbar.bg should be yellow + Then the completion model should be value Scenario: Deleting an open tab via the completion Given I have a fresh instance diff --git a/tests/end2end/features/test_completion_bdd.py b/tests/end2end/features/test_completion_bdd.py index f4ada848f..4a06bfbc0 100644 --- a/tests/end2end/features/test_completion_bdd.py +++ b/tests/end2end/features/test_completion_bdd.py @@ -24,5 +24,5 @@ bdd.scenarios('completion.feature') @bdd.then(bdd.parsers.parse("the completion model should be {model}")) def check_model(quteproc, model): """Make sure the completion model was set to something.""" - pattern = "Setting completion model to {} with pattern *".format(model) + pattern = "Starting {} completion".format(model) quteproc.wait_for(message=pattern) From 490250f5bef8cc7b2bb9cb66601fa398a7fb2d3b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 07:39:50 -0500 Subject: [PATCH 030/337] Initialize SQL for two failing tests. test_selectors and test_get_all_objects were running fine on my machine, but for some reason is failing with "Driver not loaded" on Travis. Let's try initializing SQL and see what happens. --- tests/unit/browser/webkit/test_webkitelem.py | 2 +- tests/unit/utils/test_debug.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/browser/webkit/test_webkitelem.py b/tests/unit/browser/webkit/test_webkitelem.py index deff844fe..f6cd44a8c 100644 --- a/tests/unit/browser/webkit/test_webkitelem.py +++ b/tests/unit/browser/webkit/test_webkitelem.py @@ -211,7 +211,7 @@ class TestSelectorsAndFilters: assert self.TESTS @pytest.mark.parametrize('group, val, matching', TESTS) - def test_selectors(self, webframe, group, val, matching): + def test_selectors(self, webframe, init_sql, group, val, matching): webframe.setHtml('{}'.format(val)) # Make sure setting HTML succeeded and there's a new element assert len(webframe.findAllElements('*')) == 3 diff --git a/tests/unit/utils/test_debug.py b/tests/unit/utils/test_debug.py index 9b77b9628..646858163 100644 --- a/tests/unit/utils/test_debug.py +++ b/tests/unit/utils/test_debug.py @@ -244,7 +244,7 @@ class TestGetAllObjects: def __repr__(self): return '<{}>'.format(self._name) - def test_get_all_objects(self, stubs, monkeypatch): + def test_get_all_objects(self, stubs, monkeypatch, init_sql): # pylint: disable=unused-variable widgets = [self.Object('Widget 1'), self.Object('Widget 2')] app = stubs.FakeQApplication(all_widgets=widgets) From 21757a96a3bd73e81d0365731fedc726529676ca Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 08:25:46 -0500 Subject: [PATCH 031/337] Add docstrings to SqlCompletionModel overrides. Leaving QAbstractItemModel overrides undocumented made pylint angry. --- qutebrowser/completion/models/sqlmodel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index fcd8d37c6..2c7a25bc0 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -194,6 +194,7 @@ class SqlCompletionModel(QAbstractItemModel): return self.createIndex(row, 0, None) def rowCount(self, parent=QModelIndex()): + """Override QAbstractItemModel::rowCount.""" if not parent.isValid(): # top-level return len(self._categories) @@ -206,6 +207,7 @@ class SqlCompletionModel(QAbstractItemModel): return cat.rowCount() def columnCount(self, parent=QModelIndex()): + """Override QAbstractItemModel::columnCount.""" # pylint: disable=unused-argument return 3 From d658378af360765ff11fb416b6bf9f0f9ff160c5 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 17:04:10 -0500 Subject: [PATCH 032/337] Eliminate test interference from webkit history. Initializing the qtwebkit history backend left some global state that was leaking into other tests. --- tests/unit/browser/webkit/test_history.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 844c20fb1..43b30ed19 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -362,3 +362,6 @@ def test_init(backend, qapp, tmpdir, monkeypatch): assert default_interface is None objreg.delete('web-history') + if backend == usertypes.Backend.QtWebKit: + # prevent interference with future tests + QWebHistoryInterface.setDefaultInterface(None) From be38e181a86f1adb8464de6079979ffe45d9171c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 17:06:16 -0500 Subject: [PATCH 033/337] Install libqt5sql5-sqlite for debian CI. Needed for tests to pass with the new SQL dependency. --- scripts/dev/ci/travis_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index aa4c1c6c8..b200fb6d0 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -102,7 +102,7 @@ elif [[ $TRAVIS_OS_NAME == osx ]]; then exit 0 fi -pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit python3-pyqt5.qtsql" +pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit python3-pyqt5.qtsql libqt5sql5-sqlite" pip_install pip pip_install -r misc/requirements/requirements-tox.txt From 788babbb61f8b06f307a548622a2deea6af65bf9 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 20:32:03 -0500 Subject: [PATCH 034/337] Further prevent state leakage from test_init. test_history.test_init also leaked state by leaving the instantiated history as the parent of the QApp, which was causing test_debug to fail because it was trying to dump the history object left from test_history. --- tests/unit/browser/webkit/test_history.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 43b30ed19..890ff9a89 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -361,7 +361,8 @@ def test_init(backend, qapp, tmpdir, monkeypatch): # before (so we need to test webengine before webkit) assert default_interface is None + # prevent interference with future tests objreg.delete('web-history') + hist.setParent(None) if backend == usertypes.Backend.QtWebKit: - # prevent interference with future tests QWebHistoryInterface.setDefaultInterface(None) From fc5fd6096ab546a9bc1ad7208b710c48a2a7145b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Feb 2017 20:34:14 -0500 Subject: [PATCH 035/337] Revert "Initialize SQL for two failing tests." This reverts commit 386e227ce7534f1e427db7ba6d4e53dc153a49f3. The problem was really state leakage, initializing sql for these tests isn't necessary. --- tests/unit/browser/webkit/test_webkitelem.py | 2 +- tests/unit/utils/test_debug.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/browser/webkit/test_webkitelem.py b/tests/unit/browser/webkit/test_webkitelem.py index f6cd44a8c..deff844fe 100644 --- a/tests/unit/browser/webkit/test_webkitelem.py +++ b/tests/unit/browser/webkit/test_webkitelem.py @@ -211,7 +211,7 @@ class TestSelectorsAndFilters: assert self.TESTS @pytest.mark.parametrize('group, val, matching', TESTS) - def test_selectors(self, webframe, init_sql, group, val, matching): + def test_selectors(self, webframe, group, val, matching): webframe.setHtml('{}'.format(val)) # Make sure setting HTML succeeded and there's a new element assert len(webframe.findAllElements('*')) == 3 diff --git a/tests/unit/utils/test_debug.py b/tests/unit/utils/test_debug.py index 646858163..9b77b9628 100644 --- a/tests/unit/utils/test_debug.py +++ b/tests/unit/utils/test_debug.py @@ -244,7 +244,7 @@ class TestGetAllObjects: def __repr__(self): return '<{}>'.format(self._name) - def test_get_all_objects(self, stubs, monkeypatch, init_sql): + def test_get_all_objects(self, stubs, monkeypatch): # pylint: disable=unused-variable widgets = [self.Object('Widget 1'), self.Object('Widget 2')] app = stubs.FakeQApplication(all_widgets=widgets) From 6cc2095221e428b9f52b81a48043bce663771ddc Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 17 Feb 2017 08:13:36 -0500 Subject: [PATCH 036/337] Avoid keyconf circular import. The new function-based completion API introduced a circular import: config -> keyconf -> miscmodels -> config. config only depended on keyconf so it could initialize it as part of config.init. This can be resolved by moving this to keyconf.init and initializing keyconf as part of app.init. --- qutebrowser/app.py | 5 ++++ qutebrowser/config/config.py | 1 - qutebrowser/config/parsers/keyconf.py | 33 ++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 8b2aa7d69..44178bae3 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -45,6 +45,7 @@ import qutebrowser.resources from qutebrowser.completion.models import miscmodels from qutebrowser.commands import cmdutils, runners, cmdexc from qutebrowser.config import style, config, websettings, configexc +from qutebrowser.config.parsers import keyconf from qutebrowser.browser import (urlmarks, adblock, history, browsertab, downloads) from qutebrowser.browser.network import proxy @@ -423,6 +424,10 @@ def _init_modules(args, crash_handler): config.init(qApp) save_manager.init_autosave() + log.init.debug("Initializing keys...") + keyconf.init(qApp) + save_manager.init_autosave() + log.init.debug("Initializing sql...") sql.init() diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index dd71f5333..0b1893fc0 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -249,7 +249,6 @@ def init(parent=None): parent: The parent to pass to QObjects which get initialized. """ _init_main_config(parent) - _init_key_config(parent) _init_misc() diff --git a/qutebrowser/config/parsers/keyconf.py b/qutebrowser/config/parsers/keyconf.py index 5a780a786..751eafb71 100644 --- a/qutebrowser/config/parsers/keyconf.py +++ b/qutebrowser/config/parsers/keyconf.py @@ -22,15 +22,46 @@ import collections import os.path import itertools +import sys from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.config import configdata, textwrapper from qutebrowser.commands import cmdutils, cmdexc -from qutebrowser.utils import log, utils, qtutils, message, usertypes +from qutebrowser.utils import (log, utils, qtutils, message, usertypes, objreg, + standarddir, error) from qutebrowser.completion.models import miscmodels +def init(parent=None): + """Read and save keybindings. + + Args: + parent: The parent to use for the KeyConfigParser. + """ + args = objreg.get('args') + try: + key_config = KeyConfigParser(standarddir.config(), 'keys.conf', + args.relaxed_config, parent=parent) + except (KeyConfigError, UnicodeDecodeError) as e: + log.init.exception(e) + errstr = "Error while reading key config:\n" + if e.lineno is not None: + errstr += "In line {}: ".format(e.lineno) + error.handle_fatal_exc(e, args, "Error while reading key config!", + pre_text=errstr) + # We didn't really initialize much so far, so we just quit hard. + sys.exit(usertypes.Exit.err_key_config) + else: + objreg.register('key-config', key_config) + save_manager = objreg.get('save-manager') + filename = os.path.join(standarddir.config(), 'keys.conf') + save_manager.add_saveable( + 'key-config', key_config.save, key_config.config_dirty, + config_opt=('general', 'auto-save-config'), filename=filename, + dirty=key_config.is_dirty) + + class KeyConfigError(Exception): """Raised on errors with the key config. From 6a8b1d51fa7326ac8080217c0bcfaf3d78c53cd7 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 19 Feb 2017 20:58:10 -0500 Subject: [PATCH 037/337] Avoid config -> configmodel circular import. Avoid the config dependency by using objreg.get('config') instead of config.get. --- qutebrowser/completion/models/configmodel.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index 406d8d572..7c7e3a0c6 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -19,8 +19,9 @@ """Functions that return config-related completion models.""" -from qutebrowser.config import config, configdata, configexc +from qutebrowser.config import configdata, configexc from qutebrowser.completion.models import base +from qutebrowser.utils import objreg def section(): @@ -54,6 +55,7 @@ def option(sectname): desc = "" else: desc = desc.splitlines()[0] + config = objreg.get('config') val = config.get(sectname, name, raw=True) model.new_item(cat, name, desc, val) return model @@ -68,6 +70,7 @@ def value(sectname, optname): """ model = base.CompletionModel(column_widths=(20, 70, 10)) cur_cat = model.new_category("Current/Default") + config = objreg.get('config') try: val = config.get(sectname, optname, raw=True) or '""' except (configexc.NoSectionError, configexc.NoOptionError): From 1474e38eec4ca91dd00f5b7bce60eaf86e0048b0 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 19 Feb 2017 21:36:02 -0500 Subject: [PATCH 038/337] Add urlmodel to perfect_files --- scripts/dev/check_coverage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 7e5a2d47a..3b1611a1c 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -156,6 +156,8 @@ PERFECT_FILES = [ ('tests/unit/completion/test_models.py', 'completion/models/base.py'), + ('tests/unit/completion/test_models.py', + 'completion/models/urlmodel.py'), ('tests/unit/completion/test_sortfilter.py', 'completion/models/sortfilter.py'), From c6645d47bab522a9aa9869fe19fb9ff93e697c01 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 19 Feb 2017 21:39:15 -0500 Subject: [PATCH 039/337] Remove newest_slice and StatusBar._option. newest_slice is no longer needed after the completion refactor. Now that history is based on the SQL backend, LIMIT is used instead. StatusBar._option is not used, though I'm not sure why vulture only caught it now. --- qutebrowser/utils/utils.py | 19 ------------------- tests/unit/utils/test_utils.py | 31 ------------------------------- 2 files changed, 50 deletions(-) diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py index 0b527637e..9d1320995 100644 --- a/qutebrowser/utils/utils.py +++ b/qutebrowser/utils/utils.py @@ -737,25 +737,6 @@ def sanitize_filename(name, replacement='_'): return name -def newest_slice(iterable, count): - """Get an iterable for the n newest items of the given iterable. - - Args: - count: How many elements to get. - 0: get no items: - n: get the n newest items - -1: get all items - """ - if count < -1: - raise ValueError("count can't be smaller than -1!") - elif count == 0: - return [] - elif count == -1 or len(iterable) < count: - return iterable - else: - return itertools.islice(iterable, len(iterable) - count, len(iterable)) - - def set_clipboard(data, selection=False): """Set the clipboard to some given data.""" if selection and not supports_selection(): diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index 4b03dcf8c..47595ebf5 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -759,37 +759,6 @@ def test_sanitize_filename_empty_replacement(): assert utils.sanitize_filename(name, replacement=None) == 'Bad File' -class TestNewestSlice: - - """Test newest_slice.""" - - def test_count_minus_two(self): - """Test with a count of -2.""" - with pytest.raises(ValueError): - utils.newest_slice([], -2) - - @pytest.mark.parametrize('items, count, expected', [ - # Count of -1 (all elements). - (range(20), -1, range(20)), - # Count of 0 (no elements). - (range(20), 0, []), - # Count which is much smaller than the iterable. - (range(20), 5, [15, 16, 17, 18, 19]), - # Count which is exactly one smaller.""" - (range(5), 4, [1, 2, 3, 4]), - # Count which is just as large as the iterable.""" - (range(5), 5, range(5)), - # Count which is one bigger than the iterable. - (range(5), 6, range(5)), - # Count which is much bigger than the iterable. - (range(5), 50, range(5)), - ]) - def test_good(self, items, count, expected): - """Test slices which shouldn't raise an exception.""" - sliced = utils.newest_slice(items, count) - assert list(sliced) == list(expected) - - class TestGetSetClipboard: @pytest.fixture(autouse=True) From 496859007595874e8f93a1b14ce641460531f4b6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 20 Feb 2017 08:17:21 -0500 Subject: [PATCH 040/337] Mention needed SQL packages in INSTALL. With the new SQL completion backend, some distros must install sql packages. This also removes trailing whitespace from one line. --- INSTALL.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.asciidoc b/INSTALL.asciidoc index ad4b8446f..48427e121 100644 --- a/INSTALL.asciidoc +++ b/INSTALL.asciidoc @@ -27,7 +27,7 @@ Using the packages Install the dependencies via apt-get: ---- -# apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-jinja2 python3-pygments python3-yaml +# apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-jinja2 python3-pygments python3-yaml python3-pyqt5.qtsql libqt5sql5-sqlite ---- On Debian Stretch or Ubuntu 17.04 or later, it's also recommended to use the @@ -53,7 +53,7 @@ Build it from git Install the dependencies via apt-get: ---- -# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python-tox python3-sip python3-dev +# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python-tox python3-sip python3-dev python3-pyqt5.qtsql libqt5sql5-sqlite ---- On Debian Stretch or Ubuntu 17.04 or later, it's also recommended to install From a050cb94f675779986a683a3216890ab21bf3803 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 20 Feb 2017 08:33:38 -0500 Subject: [PATCH 041/337] Report sqlite version with --version. --- qutebrowser/misc/sql.py | 7 +++++++ qutebrowser/utils/version.py | 6 +++++- tests/unit/misc/test_sql.py | 3 +++ tests/unit/utils/test_version.py | 2 ++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index c051acf03..e0b55dbd2 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -41,6 +41,13 @@ def close(): QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) +def version(): + """Return the sqlite version string.""" + result = run_query("select sqlite_version()") + result.next() + return result.record().value(0) + + def run_query(querystr, values=None): """Run the given SQL query string on the database. diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index 08779e097..ac74f997a 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -45,7 +45,7 @@ except ImportError: # pragma: no cover import qutebrowser from qutebrowser.utils import log, utils, standarddir, usertypes, qtutils -from qutebrowser.misc import objects, earlyinit +from qutebrowser.misc import objects, earlyinit, sql from qutebrowser.browser import pdfjs @@ -327,6 +327,10 @@ def version(): lines += ['pdf.js: {}'.format(_pdfjs_version())] + sql.init() + lines += ['sqlite: {}'.format(sql.version())] + sql.close() + lines += [ 'SSL: {}'.format(QSslSocket.sslLibraryVersionString()), '', diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 4a5299b48..b24f2fbcf 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -120,3 +120,6 @@ def test_delete_all(qtbot): with qtbot.waitSignal(table.changed): table.delete_all() assert list(table) == [] + +def test_version(): + assert isinstance(sql.version(), str) diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 54ee2cdd0..8c0f16081 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -834,6 +834,7 @@ def test_version_output(git_commit, frozen, style, with_webkit, 'QApplication': (stubs.FakeQApplication(style='STYLE') if style else stubs.FakeQApplication(instance=None)), 'QLibraryInfo.location': (lambda _loc: 'QT PATH'), + 'sql.version': lambda: 'SQLITE VERSION', } substitutions = { @@ -892,6 +893,7 @@ def test_version_output(git_commit, frozen, style, with_webkit, MODULE VERSION 1 MODULE VERSION 2 pdf.js: PDFJS VERSION + sqlite: SQLITE VERSION SSL: SSL VERSION {style} Platform: PLATFORM, ARCHITECTURE{linuxdist} From 3c676f9562606bcddaa2291531ff488c44880b9b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 20 Feb 2017 12:15:06 -0500 Subject: [PATCH 042/337] Fix pylint/flake8 errors for SQL work. --- qutebrowser/utils/utils.py | 1 - tests/unit/misc/test_sql.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py index 9d1320995..a2c9c104d 100644 --- a/qutebrowser/utils/utils.py +++ b/qutebrowser/utils/utils.py @@ -28,7 +28,6 @@ import os.path import collections import functools import contextlib -import itertools import socket import shlex diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index b24f2fbcf..8bf7ddb4b 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -121,5 +121,6 @@ def test_delete_all(qtbot): table.delete_all() assert list(table) == [] + def test_version(): assert isinstance(sql.version(), str) From 56f3b3a02709736ba702b3b3ef4342e3a4c98a42 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 21 Feb 2017 08:27:52 -0500 Subject: [PATCH 043/337] Small review fixups for SQL implementation. Respond to the low-hanging code review fruit: - Clean up some comments - Remove an acidentally added duplicate init_autosave - Combine two test_history tests - Move test_init cleanup into a fixture to ensure it gets called. - Name the _ argument of bind(_) to _key - Ensure index is valid for first_item/last_item - Move SqlException to top of module - Rename test_index to test_getitem - Return QItemFlags.None instead of None - Fix copyright dates (its 2017 now!) - Use * to force some args to be keyword-only - Make some returns explicit - Add sql to LOGGER_NAMES - Add a comment to explain the sql escape statement --- qutebrowser/app.py | 1 - qutebrowser/browser/urlmarks.py | 12 ++---- qutebrowser/completion/models/base.py | 3 +- qutebrowser/completion/models/miscmodels.py | 4 +- qutebrowser/completion/models/sqlmodel.py | 34 +++++++++------- qutebrowser/misc/sql.py | 19 ++++----- qutebrowser/utils/log.py | 2 +- tests/helpers/fixtures.py | 2 +- tests/unit/browser/webkit/test_history.py | 44 +++++++++++---------- tests/unit/completion/test_sqlmodel.py | 2 +- tests/unit/misc/test_sql.py | 4 +- 11 files changed, 65 insertions(+), 62 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 44178bae3..33726b058 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -426,7 +426,6 @@ def _init_modules(args, crash_handler): log.init.debug("Initializing keys...") keyconf.init(qApp) - save_manager.init_autosave() log.init.debug("Initializing sql...") sql.init() diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index 5f7a4fb77..580d77881 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -104,10 +104,7 @@ class UrlMarkManager(sql.SqlTable): class QuickmarkManager(UrlMarkManager): - """Manager for quickmarks. - - The primary key for quickmarks is their *name*. - """ + """Manager for quickmarks.""" def __init__(self, parent=None): super().__init__('Quickmarks', ['name', 'url'], primary_key='name', @@ -179,7 +176,7 @@ class QuickmarkManager(UrlMarkManager): set_mark() def delete_by_qurl(self, url): - """Look up a quickmark by QUrl, returning its name.""" + """Delete a quickmark by QUrl (as opposed to by name).""" qtutils.ensure_valid(url) urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) try: @@ -204,10 +201,7 @@ class QuickmarkManager(UrlMarkManager): class BookmarkManager(UrlMarkManager): - """Manager for bookmarks. - - The primary key for bookmarks is their *url*. - """ + """Manager for bookmarks.""" def __init__(self, parent=None): super().__init__('Bookmarks', ['url', 'title'], primary_key='url', diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py index 112c6ed00..f5148ff9c 100644 --- a/qutebrowser/completion/models/base.py +++ b/qutebrowser/completion/models/base.py @@ -36,9 +36,10 @@ class CompletionModel(QStandardItemModel): Attributes: column_widths: The width percentages of the columns used in the + completion view. """ - def __init__(self, column_widths=(30, 70, 0), columns_to_filter=None, + def __init__(self, *, column_widths=(30, 70, 0), columns_to_filter=None, delete_cur_item=None, parent=None): super().__init__(parent) self.setColumnCount(3) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 8bb234521..f5ed72f5d 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -130,11 +130,11 @@ def buffer(): return model -def bind(_): +def bind(_key): """A CompletionModel filled with all bindable commands and descriptions. Args: - _: the key being bound. + _key: the key being bound. """ # TODO: offer a 'Current binding' completion based on the key. model = base.CompletionModel(column_widths=(20, 60, 20)) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 2c7a25bc0..1053fb2f2 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2016 Ryan Roden-Corrent (rcorre) +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # @@ -24,12 +24,14 @@ import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from PyQt5.QtSql import QSqlQueryModel -from qutebrowser.utils import log +from qutebrowser.utils import log, qtutils from qutebrowser.misc import sql class _SqlCompletionCategory(QSqlQueryModel): - def __init__(self, name, sort_by, sort_order, limit, select, where, + """Wraps a SqlQuery for use as a completion category.""" + + def __init__(self, name, *, sort_by, sort_order, limit, select, where, columns_to_filter, parent=None): super().__init__(parent=parent) self.tablename = name @@ -38,6 +40,8 @@ class _SqlCompletionCategory(QSqlQueryModel): self._fields = [query.record().fieldName(i) for i in columns_to_filter] querystr = 'select {} from {} where ('.format(select, name) + # the incoming pattern will have literal % and _ escaped with '\' + # we need to tell sql to treat '\' as an escape character querystr += ' or '.join("{} like ? escape '\\'".format(f) for f in self._fields) querystr += ')' @@ -45,6 +49,7 @@ class _SqlCompletionCategory(QSqlQueryModel): querystr += ' and ' + where if sort_by: + assert sort_order is not None sortstr = 'asc' if sort_order == Qt.AscendingOrder else 'desc' querystr += ' order by {} {}'.format(sort_by, sortstr) @@ -69,19 +74,15 @@ class SqlCompletionModel(QAbstractItemModel): Top level indices represent categories, each of which is backed by a single table. Child indices represent rows of those tables. - Class Attributes: - COLUMN_WIDTHS: The width percentages of the columns used in the - completion view. - Attributes: column_widths: The width percentages of the columns used in the - completion view. + completion view. columns_to_filter: A list of indices of columns to apply the filter to. pattern: Current filter pattern, used for highlighting. _categories: The category tables. """ - def __init__(self, column_widths=(30, 70, 0), columns_to_filter=None, + def __init__(self, *, column_widths=(30, 70, 0), columns_to_filter=None, delete_cur_item=None, parent=None): super().__init__(parent) self.columns_to_filter = columns_to_filter or [0] @@ -103,8 +104,9 @@ class SqlCompletionModel(QAbstractItemModel): # categories have an empty internalPointer if index.isValid() and not index.internalPointer(): return self._categories[index.row()] + return None - def new_category(self, name, select='*', where=None, sort_by=None, + def new_category(self, name, *, select='*', where=None, sort_by=None, sort_order=None, limit=None): """Create a new completion category and add it to this model. @@ -135,7 +137,7 @@ class SqlCompletionModel(QAbstractItemModel): Return: The item data, or None on an invalid index. """ if not index.isValid() or role != Qt.DisplayRole: - return + return None if not index.parent().isValid(): if index.column() == 0: return self._categories[index.row()].tablename @@ -152,7 +154,7 @@ class SqlCompletionModel(QAbstractItemModel): Return: The item flags, or Qt.NoItemFlags on error. """ if not index.isValid(): - return + return Qt.NoItemFlags if index.parent().isValid(): # item return (Qt.ItemIsEnabled | Qt.ItemIsSelectable | @@ -253,7 +255,9 @@ class SqlCompletionModel(QAbstractItemModel): for row, table in enumerate(self._categories): if table.rowCount() > 0: parent = self.index(row, 0) - return self.index(0, 0, parent) + index = self.index(0, 0, parent) + qtutils.ensure_valid(index) + return index return QModelIndex() def last_item(self): @@ -262,7 +266,9 @@ class SqlCompletionModel(QAbstractItemModel): childcount = table.rowCount() if childcount > 0: parent = self.index(row, 0) - return self.index(childcount - 1, 0, parent) + index = self.index(childcount - 1, 0, parent) + qtutils.ensure_valid(index) + return index return QModelIndex() diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index e0b55dbd2..c9249d362 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2016 Ryan Roden-Corrent (rcorre) +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # @@ -27,13 +27,21 @@ from qutebrowser.utils import log import collections +class SqlException(Exception): + + """Raised on an error interacting with the SQL database.""" + + pass + + def init(): """Initialize the SQL database connection.""" database = QSqlDatabase.addDatabase('QSQLITE') # In-memory database, see https://sqlite.org/inmemorydb.html database.setDatabaseName(':memory:') if not database.open(): - raise SqlException("Failed to open in-memory sqlite database") + raise SqlException("Failed to open in-memory sqlite database: {}" + .format(database.lastError().text())) def close(): @@ -169,10 +177,3 @@ class SqlTable(QObject): """Remove all row from the table.""" run_query("DELETE FROM {}".format(self._name)) self.changed.emit() - - -class SqlException(Exception): - - """Raised on an error interacting with the SQL database.""" - - pass diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py index 513c05062..6cdc61f41 100644 --- a/qutebrowser/utils/log.py +++ b/qutebrowser/utils/log.py @@ -94,7 +94,7 @@ LOGGER_NAMES = [ 'commands', 'signals', 'downloads', 'js', 'qt', 'rfc6266', 'ipc', 'shlexer', 'save', 'message', 'config', 'sessions', - 'webelem', 'prompt', 'network' + 'webelem', 'prompt', 'network', 'sql' ] diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 7fc05a578..0b69ad89f 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -457,7 +457,7 @@ def short_tmpdir(): yield py.path.local(tdir) # pylint: disable=no-member -@pytest.fixture() +@pytest.fixture def init_sql(): """Initialize the SQL module, and shut it down after the test.""" sql.init() diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 890ff9a89..9d8711152 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -193,21 +193,15 @@ def test_clear(qtbot, tmpdir): assert lines == ['67890 http://www.the-compiler.org/'] -def test_add_item(qtbot, hist): +@pytest.mark.parametrize('item', [ + ('http://www.example.com', 12346, 'the title', False), + ('http://www.example.com', 12346, 'the title', True) +]) +def test_add_item(qtbot, hist, item): list(hist.async_read()) - url = 'http://www.example.com/' - - hist.add_url(QUrl(url), atime=12345, title="the title") - - assert hist[url] == (url, 'the title', 12345, False) - - -def test_add_item_redirect(qtbot, hist): - list(hist.async_read()) - url = 'http://www.example.com/' - hist.add_url(QUrl(url), redirect=True, atime=12345) - - assert hist[url] == (url, '', 12345, True) + (url, atime, title, redirect) = item + hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) + assert hist[url] == (url, title, atime, redirect) def test_add_item_redirect_update(qtbot, tmpdir, fake_save_manager): @@ -329,9 +323,23 @@ def test_history_interface(qtbot, webview, hist_interface): webview.load(url) +@pytest.fixture +def cleanup_init(): + yield + # prevent test_init from leaking state + hist = objreg.get('web-history') + hist.setParent(None) + objreg.delete('web-history') + try: + from PyQt5.QtWebKit import QWebHistoryInterface + QWebHistoryInterface.setDefaultInterface(None) + except: + pass + + @pytest.mark.parametrize('backend', [usertypes.Backend.QtWebEngine, usertypes.Backend.QtWebKit]) -def test_init(backend, qapp, tmpdir, monkeypatch): +def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): if backend == usertypes.Backend.QtWebKit: pytest.importorskip('PyQt5.QtWebKitWidgets') else: @@ -360,9 +368,3 @@ def test_init(backend, qapp, tmpdir, monkeypatch): # For this to work, nothing can ever have called setDefaultInterface # before (so we need to test webengine before webkit) assert default_interface is None - - # prevent interference with future tests - objreg.delete('web-history') - hist.setParent(None) - if backend == usertypes.Backend.QtWebKit: - QWebHistoryInterface.setDefaultInterface(None) diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index f6daa722c..044eecb35 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2016 Ryan Roden-Corrent (rcorre) +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 8bf7ddb4b..f4898b25d 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2016 Ryan Roden-Corrent (rcorre) +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # @@ -102,7 +102,7 @@ def test_contains(): assert 'thirteen' in table -def test_index(): +def test_getitem(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') table.insert(['one', 1, False]) table.insert(['nine', 9, False]) From c4c5723a61f170cb5945b49ce18b46c5b477ff42 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 21 Feb 2017 09:05:22 -0500 Subject: [PATCH 044/337] Sort history completion entries by atime. --- qutebrowser/completion/models/sqlmodel.py | 7 +++---- qutebrowser/completion/models/urlmodel.py | 2 +- tests/unit/completion/test_sqlmodel.py | 14 +++++++------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 1053fb2f2..99c47cf28 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -49,9 +49,8 @@ class _SqlCompletionCategory(QSqlQueryModel): querystr += ' and ' + where if sort_by: - assert sort_order is not None - sortstr = 'asc' if sort_order == Qt.AscendingOrder else 'desc' - querystr += ' order by {} {}'.format(sort_by, sortstr) + assert sort_order == 'asc' or sort_order == 'desc' + querystr += ' order by {} {}'.format(sort_by, sort_order) if limit: querystr += ' limit {}'.format(limit) @@ -115,7 +114,7 @@ class SqlCompletionModel(QAbstractItemModel): select: A custom result column expression for the select statement. where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. - sort_order: Sorting order, if sort_by is non-None. + sort_order: Either 'asc' or 'desc', if sort_by is non-None limit: Maximum row count to return on a query. Return: A new CompletionCategory. diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 387603dd5..6a10d9c44 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -70,7 +70,7 @@ def url(): model.new_category('Quickmarks', select='url, name') model.new_category('Bookmarks') model.new_category('History', - limit=limit, + limit=limit, sort_order='desc', sort_by='atime', select='url, title, {}'.format(select_time), where='not redirect') return model diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 044eecb35..ab14bce75 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -73,31 +73,31 @@ def test_count(rowcounts, expected): @pytest.mark.parametrize('sort_by, sort_order, data, expected', [ - (None, Qt.AscendingOrder, + (None, 'asc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), - ('a', Qt.AscendingOrder, + ('a', 'asc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - ('a', Qt.DescendingOrder, + ('a', 'desc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - ('b', Qt.AscendingOrder, + ('b', 'asc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - ('b', Qt.DescendingOrder, + ('b', 'desc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - ('c', Qt.AscendingOrder, + ('c', 'asc', [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], [('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]), - ('c', Qt.DescendingOrder, + ('c', 'desc', [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), ]) From 99c9b2d3963701fd598a7d036096c73183f778c6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 22 Feb 2017 21:05:03 -0500 Subject: [PATCH 045/337] Fix two small mistakes after SQL code review. urlmodel is now sorted, so the test had to be adjusted. Also remove one unused import. --- tests/unit/completion/test_models.py | 2 +- tests/unit/completion/test_sqlmodel.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index ddc746aa6..b866e6439 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -297,7 +297,7 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ], "History": [ - ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ('https://github.com', 'https://github.com', '2016-05-01'), ('https://python.org', 'Welcome to Python.org', '2016-03-08'), ], }) diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index ab14bce75..49dc96ccf 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -20,7 +20,6 @@ """Tests for the base sql completion model.""" import pytest -from PyQt5.QtCore import Qt from qutebrowser.misc import sql from qutebrowser.completion.models import sqlmodel From 921211bbaa09904fbeddf882cbe5f4d1ed1bb1ec Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 22 Feb 2017 21:12:23 -0500 Subject: [PATCH 046/337] Remove web-history-max-items. This was a performance optimization that shouldn't be needed with the new SQL history backend. This also removes support for the LIMIT feature from SqlTable as it only existed to support web-history-max-items. --- qutebrowser/completion/models/sqlmodel.py | 10 +++------- qutebrowser/completion/models/urlmodel.py | 3 +-- qutebrowser/config/config.py | 1 + qutebrowser/config/configdata.py | 5 ----- tests/unit/completion/test_models.py | 14 +++++--------- tests/unit/completion/test_sqlmodel.py | 9 --------- 6 files changed, 10 insertions(+), 32 deletions(-) diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/sqlmodel.py index 99c47cf28..e3e5a5c5f 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/sqlmodel.py @@ -31,7 +31,7 @@ from qutebrowser.misc import sql class _SqlCompletionCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" - def __init__(self, name, *, sort_by, sort_order, limit, select, where, + def __init__(self, name, *, sort_by, sort_order, select, where, columns_to_filter, parent=None): super().__init__(parent=parent) self.tablename = name @@ -52,9 +52,6 @@ class _SqlCompletionCategory(QSqlQueryModel): assert sort_order == 'asc' or sort_order == 'desc' querystr += ' order by {} {}'.format(sort_by, sort_order) - if limit: - querystr += ' limit {}'.format(limit) - self._querystr = querystr self.set_pattern('%') @@ -106,7 +103,7 @@ class SqlCompletionModel(QAbstractItemModel): return None def new_category(self, name, *, select='*', where=None, sort_by=None, - sort_order=None, limit=None): + sort_order=None): """Create a new completion category and add it to this model. Args: @@ -115,12 +112,11 @@ class SqlCompletionModel(QAbstractItemModel): where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Either 'asc' or 'desc', if sort_by is non-None - limit: Maximum row count to return on a query. Return: A new CompletionCategory. """ cat = _SqlCompletionCategory(name, parent=self, sort_by=sort_by, - sort_order=sort_order, limit=limit, + sort_order=sort_order, select=select, where=where, columns_to_filter=self.columns_to_filter) self._categories.append(cat) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 6a10d9c44..0b357c3d7 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -64,13 +64,12 @@ def url(): model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), columns_to_filter=[_URLCOL, _TEXTCOL], delete_cur_item=_delete_url) - limit = config.get('completion', 'web-history-max-items') timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) model.new_category('Quickmarks', select='url, name') model.new_category('Bookmarks') model.new_category('History', - limit=limit, sort_order='desc', sort_by='atime', + sort_order='desc', sort_by='atime', select='url, title, {}'.format(select_time), where='not redirect') return model diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index 0b1893fc0..e7e74cc19 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -414,6 +414,7 @@ class ConfigManager(QObject): ('storage', 'offline-storage-default-quota'), ('storage', 'offline-web-application-cache-quota'), ('content', 'css-regions'), + ('completion', 'web-history-max-items'), ] CHANGED_OPTIONS = { ('content', 'cookies-accept'): diff --git a/qutebrowser/config/configdata.py b/qutebrowser/config/configdata.py index 28cf41ac5..fd3235c8b 100644 --- a/qutebrowser/config/configdata.py +++ b/qutebrowser/config/configdata.py @@ -506,11 +506,6 @@ def data(readonly=False): "How many commands to save in the command history.\n\n" "0: no history / -1: unlimited"), - ('web-history-max-items', - SettingValue(typ.Int(minval=-1), '1000'), - "How many URLs to show in the web history.\n\n" - "0: no history / -1: unlimited"), - ('quick-complete', SettingValue(typ.Bool(), 'true'), "Whether to move on to the next part when there's only one " diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index b866e6439..224d7ed59 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -274,13 +274,10 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, Verify that: - quickmarks, bookmarks, and urls are included - - no more than 'web-history-max-items' items are included (TODO) - - the most recent entries are included + - entries are sorted by access time - redirect entries are not included """ - # TODO: time formatting and item limiting - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - 'web-history-max-items': 2} + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -299,6 +296,7 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, "History": [ ('https://github.com', 'https://github.com', '2016-05-01'), ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), ], }) @@ -307,8 +305,7 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): """Test deleting a bookmark from the url completion model.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - 'web-history-max-items': 2} + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -325,8 +322,7 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): """Test deleting a bookmark from the url completion model.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - 'web-history-max-items': 2} + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py index 49dc96ccf..8b9727dfd 100644 --- a/tests/unit/completion/test_sqlmodel.py +++ b/tests/unit/completion/test_sqlmodel.py @@ -206,15 +206,6 @@ def test_first_last_item(data, first, last): assert model.data(model.last_item()) == last -def test_limit(): - table = sql.SqlTable('test_limit', ['a'], primary_key='a') - for i in range(5): - table.insert([i]) - model = sqlmodel.SqlCompletionModel() - model.new_category('test_limit', limit=3) - assert model.count() == 3 - - def test_select(): table = sql.SqlTable('test_select', ['a', 'b', 'c'], primary_key='a') table.insert(['foo', 'bar', 'baz']) From e3a33ca42744b374f01cfd8792fdac845e3b22e0 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 22 Feb 2017 22:25:11 -0500 Subject: [PATCH 047/337] Implement a hybrid list/sql completion model. Now all completion models are of a single type called CompletionModel. This model combines one or more categories. A category can either be a ListCategory or a SqlCategory. This simplifies the API, and will allow the use of models that combine simple list-based and sql sources. This is important for two reasons: - Adding searchengines to url completion - Using an on-disk sqlite database for history, while keeping bookmarks and quickmars as text files. --- qutebrowser/completion/completer.py | 24 +-- qutebrowser/completion/completiondelegate.py | 2 +- qutebrowser/completion/completionwidget.py | 6 +- qutebrowser/completion/models/base.py | 105 ------------- .../{sqlmodel.py => completionmodel.py} | 126 ++++++--------- qutebrowser/completion/models/configmodel.py | 37 ++--- qutebrowser/completion/models/listcategory.py | 66 ++++++++ qutebrowser/completion/models/miscmodels.py | 58 ++++--- qutebrowser/completion/models/sortfilter.py | 43 +----- qutebrowser/completion/models/sqlcategory.py | 64 ++++++++ qutebrowser/completion/models/urlmodel.py | 16 +- tests/unit/completion/test_completionmodel.py | 146 ++++++++++++++++++ .../unit/completion/test_completionwidget.py | 52 +++---- tests/unit/completion/test_models.py | 30 ++-- tests/unit/completion/test_sortfilter.py | 58 +------ 15 files changed, 426 insertions(+), 407 deletions(-) delete mode 100644 qutebrowser/completion/models/base.py rename qutebrowser/completion/models/{sqlmodel.py => completionmodel.py} (63%) create mode 100644 qutebrowser/completion/models/listcategory.py create mode 100644 qutebrowser/completion/models/sqlcategory.py create mode 100644 tests/unit/completion/test_completionmodel.py diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index f403354c3..94c7f9352 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -24,7 +24,7 @@ from PyQt5.QtCore import pyqtSlot, QObject, QTimer from qutebrowser.config import config from qutebrowser.commands import cmdutils, runners from qutebrowser.utils import log, utils -from qutebrowser.completion.models import sortfilter, miscmodels +from qutebrowser.completion.models import miscmodels class Completer(QObject): @@ -62,22 +62,6 @@ class Completer(QObject): completion = self.parent() return completion.model() - def _get_completion_model(self, completion, pos_args): - """Get a completion model based on an enum member. - - Args: - completion: A usertypes.Completion member. - pos_args: The positional args entered before the cursor. - - Return: - A completion model or None. - """ - model = completion(*pos_args) - if model is None or hasattr(model, 'set_pattern'): - return model - else: - return sortfilter.CompletionFilterModel(source=model, parent=self) - def _get_new_completion(self, before_cursor, under_cursor): """Get a new completion. @@ -96,9 +80,8 @@ class Completer(QObject): log.completion.debug("After removing flags: {}".format(before_cursor)) if not before_cursor: # '|' or 'set|' - model = miscmodels.command() log.completion.debug('Starting command completion') - return sortfilter.CompletionFilterModel(source=model, parent=self) + return miscmodels.command() try: cmd = cmdutils.cmd_dict[before_cursor[0]] except KeyError: @@ -113,7 +96,8 @@ class Completer(QObject): return None if completion is None: return None - model = self._get_completion_model(completion, before_cursor[1:]) + + model = completion(*before_cursor[1:]) log.completion.debug('Starting {} completion' .format(completion.__name__)) return model diff --git a/qutebrowser/completion/completiondelegate.py b/qutebrowser/completion/completiondelegate.py index 1d5dfadf0..776b2164c 100644 --- a/qutebrowser/completion/completiondelegate.py +++ b/qutebrowser/completion/completiondelegate.py @@ -197,7 +197,7 @@ class CompletionItemDelegate(QStyledItemDelegate): if index.parent().isValid(): pattern = index.model().pattern - columns_to_filter = index.model().srcmodel.columns_to_filter + columns_to_filter = index.model().columns_to_filter if index.column() in columns_to_filter and pattern: repl = r'\g<0>' text = re.sub(re.escape(pattern).replace(r'\ ', r'|'), diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 3d2aef27e..f6a980612 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -299,7 +299,7 @@ class CompletionView(QTreeView): if pattern is not None: model.set_pattern(pattern) - self._column_widths = model.srcmodel.column_widths + self._column_widths = model.column_widths self._resize_columns() self._maybe_update_geometry() @@ -368,7 +368,7 @@ class CompletionView(QTreeView): """Delete the current completion item.""" if not self.currentIndex().isValid(): raise cmdexc.CommandError("No item selected!") - if self.model().srcmodel.delete_cur_item is None: + if self.model().delete_cur_item is None: raise cmdexc.CommandError("Cannot delete this item.") else: - self.model().srcmodel.delete_cur_item(self) + self.model().delete_cur_item(self) diff --git a/qutebrowser/completion/models/base.py b/qutebrowser/completion/models/base.py deleted file mode 100644 index f5148ff9c..000000000 --- a/qutebrowser/completion/models/base.py +++ /dev/null @@ -1,105 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2014-2017 Florian Bruhin (The Compiler) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""The base completion model for completion in the command line. - -Module attributes: - Role: An enum of user defined model roles. -""" - -from PyQt5.QtCore import Qt -from PyQt5.QtGui import QStandardItemModel, QStandardItem - - -class CompletionModel(QStandardItemModel): - - """A simple QStandardItemModel adopted for completions. - - Used for showing completions later in the CompletionView. Supports setting - marks and adding new categories/items easily. - - Attributes: - column_widths: The width percentages of the columns used in the - completion view. - """ - - def __init__(self, *, column_widths=(30, 70, 0), columns_to_filter=None, - delete_cur_item=None, parent=None): - super().__init__(parent) - self.setColumnCount(3) - self.columns_to_filter = columns_to_filter or [0] - self.column_widths = column_widths - self.delete_cur_item = delete_cur_item - - def new_category(self, name): - """Add a new category to the model. - - Args: - name: The name of the category to add. - - Return: - The created QStandardItem. - """ - cat = QStandardItem(name) - self.appendRow(cat) - return cat - - def new_item(self, cat, name, desc='', misc=None): - """Add a new item to a category. - - Args: - cat: The parent category. - name: The name of the item. - desc: The description of the item. - misc: Misc text to display. - - Return: - A (nameitem, descitem, miscitem) tuple. - """ - assert not isinstance(name, int) - assert not isinstance(desc, int) - assert not isinstance(misc, int) - - nameitem = QStandardItem(name) - descitem = QStandardItem(desc) - miscitem = QStandardItem(misc) - - cat.appendRow([nameitem, descitem, miscitem]) - - def flags(self, index): - """Return the item flags for index. - - Override QAbstractItemModel::flags. - - Args: - index: The QModelIndex to get item flags for. - - Return: - The item flags, or Qt.NoItemFlags on error. - """ - if not index.isValid(): - return - - if index.parent().isValid(): - # item - return (Qt.ItemIsEnabled | Qt.ItemIsSelectable | - Qt.ItemNeverHasChildren) - else: - # category - return Qt.NoItemFlags diff --git a/qutebrowser/completion/models/sqlmodel.py b/qutebrowser/completion/models/completionmodel.py similarity index 63% rename from qutebrowser/completion/models/sqlmodel.py rename to qutebrowser/completion/models/completionmodel.py index e3e5a5c5f..b8dcc4601 100644 --- a/qutebrowser/completion/models/sqlmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) +# Copyright 2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # @@ -17,65 +17,29 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""A completion model backed by SQL tables.""" +"""A model that proxies access to one or more completion categories.""" import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel -from PyQt5.QtSql import QSqlQueryModel from qutebrowser.utils import log, qtutils -from qutebrowser.misc import sql +from qutebrowser.completion.models import sortfilter, listcategory, sqlcategory -class _SqlCompletionCategory(QSqlQueryModel): - """Wraps a SqlQuery for use as a completion category.""" +class CompletionModel(QAbstractItemModel): - def __init__(self, name, *, sort_by, sort_order, select, where, - columns_to_filter, parent=None): - super().__init__(parent=parent) - self.tablename = name + """A model that proxies access to one or more completion categories. - query = sql.run_query('select * from {} limit 1'.format(name)) - self._fields = [query.record().fieldName(i) for i in columns_to_filter] - - querystr = 'select {} from {} where ('.format(select, name) - # the incoming pattern will have literal % and _ escaped with '\' - # we need to tell sql to treat '\' as an escape character - querystr += ' or '.join("{} like ? escape '\\'".format(f) - for f in self._fields) - querystr += ')' - if where: - querystr += ' and ' + where - - if sort_by: - assert sort_order == 'asc' or sort_order == 'desc' - querystr += ' order by {} {}'.format(sort_by, sort_order) - - self._querystr = querystr - self.set_pattern('%') - - def set_pattern(self, pattern): - query = sql.run_query(self._querystr, [pattern for _ in self._fields]) - self.setQuery(query) - - -class SqlCompletionModel(QAbstractItemModel): - - """A sqlite-based model that provides data for the CompletionView. - - This model is a wrapper around one or more sql tables. The tables are all - stored in a single database in qutebrowser's cache directory. - - Top level indices represent categories, each of which is backed by a single - table. Child indices represent rows of those tables. + Top level indices represent categories. + Child indices represent rows of those tables. Attributes: column_widths: The width percentages of the columns used in the completion view. columns_to_filter: A list of indices of columns to apply the filter to. pattern: Current filter pattern, used for highlighting. - _categories: The category tables. + _categories: The sub-categories. """ def __init__(self, *, column_widths=(30, 70, 0), columns_to_filter=None, @@ -84,7 +48,6 @@ class SqlCompletionModel(QAbstractItemModel): self.columns_to_filter = columns_to_filter or [0] self.column_widths = column_widths self._categories = [] - self.srcmodel = self # TODO: dummy for compat with old API self.pattern = '' self.delete_cur_item = delete_cur_item @@ -94,7 +57,7 @@ class SqlCompletionModel(QAbstractItemModel): Args: idx: A QModelIndex Returns: - A _SqlCompletionCategory if the index points at one, else None + A category if the index points at one, else None """ # items hold an index to the parent category in their internalPointer # categories have an empty internalPointer @@ -102,7 +65,19 @@ class SqlCompletionModel(QAbstractItemModel): return self._categories[index.row()] return None - def new_category(self, name, *, select='*', where=None, sort_by=None, + def add_list(self, name, items): + """Add a list of items as a completion category. + + Args: + name: Title of the category. + items: List of tuples. + """ + cat = listcategory.ListCategory(name, items, parent=self) + filtermodel = sortfilter.CompletionFilterModel(cat, + self.columns_to_filter) + self._categories.append(filtermodel) + + def add_sqltable(self, name, *, select='*', where=None, sort_by=None, sort_order=None): """Create a new completion category and add it to this model. @@ -112,13 +87,11 @@ class SqlCompletionModel(QAbstractItemModel): where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Either 'asc' or 'desc', if sort_by is non-None - - Return: A new CompletionCategory. """ - cat = _SqlCompletionCategory(name, parent=self, sort_by=sort_by, - sort_order=sort_order, - select=select, where=where, - columns_to_filter=self.columns_to_filter) + cat = sqlcategory.SqlCategory(name, parent=self, sort_by=sort_by, + sort_order=sort_order, + select=select, where=where, + columns_to_filter=self.columns_to_filter) self._categories.append(cat) def data(self, index, role=Qt.DisplayRole): @@ -135,11 +108,11 @@ class SqlCompletionModel(QAbstractItemModel): return None if not index.parent().isValid(): if index.column() == 0: - return self._categories[index.row()].tablename + return self._categories[index.row()].name else: - table = self._categories[index.parent().row()] - idx = table.index(index.row(), index.column()) - return table.data(idx) + cat = self._categories[index.parent().row()] + idx = cat.index(index.row(), index.column()) + return cat.data(idx) def flags(self, index): """Return the item flags for index. @@ -171,7 +144,7 @@ class SqlCompletionModel(QAbstractItemModel): if parent.isValid(): if parent.column() != 0: return QModelIndex() - # store a pointer to the parent table in internalPointer + # store a pointer to the parent category in internalPointer return self.createIndex(row, col, self._categories[parent.row()]) return self.createIndex(row, col, None) @@ -183,11 +156,11 @@ class SqlCompletionModel(QAbstractItemModel): Args: index: The QModelIndex to get the parent index for. """ - parent_table = index.internalPointer() - if not parent_table: + parent_cat = index.internalPointer() + if not parent_cat: # categories have no parent return QModelIndex() - row = self._categories.index(parent_table) + row = self._categories.index(parent_cat) return self.createIndex(row, 0, None) def rowCount(self, parent=QModelIndex()): @@ -209,24 +182,24 @@ class SqlCompletionModel(QAbstractItemModel): return 3 def canFetchMore(self, parent): - """Override to forward the call to the tables.""" + """Override to forward the call to the categories.""" cat = self._cat_from_idx(parent) if cat: - return cat.canFetchMore() + return cat.canFetchMore(parent) return False def fetchMore(self, parent): - """Override to forward the call to the tables.""" + """Override to forward the call to the categories.""" cat = self._cat_from_idx(parent) if cat: - cat.fetchMore() + cat.fetchMore(parent) def count(self): """Return the count of non-category items.""" return sum(t.rowCount() for t in self._categories) def set_pattern(self, pattern): - """Set the filter pattern for all category tables. + """Set the filter pattern for all categories. This will apply to the fields indicated in columns_to_filter. @@ -236,19 +209,13 @@ class SqlCompletionModel(QAbstractItemModel): log.completion.debug("Setting completion pattern '{}'".format(pattern)) # TODO: should pattern be saved in the view layer instead? self.pattern = pattern - # escape to treat a user input % or _ as a literal, not a wildcard - pattern = pattern.replace('%', '\\%') - pattern = pattern.replace('_', '\\_') - # treat spaces as wildcards to match any of the typed words - pattern = re.sub(r' +', '%', pattern) - pattern = '%{}%'.format(pattern) for cat in self._categories: cat.set_pattern(pattern) def first_item(self): """Return the index of the first child (non-category) in the model.""" - for row, table in enumerate(self._categories): - if table.rowCount() > 0: + for row, cat in enumerate(self._categories): + if cat.rowCount() > 0: parent = self.index(row, 0) index = self.index(0, 0, parent) qtutils.ensure_valid(index) @@ -257,18 +224,11 @@ class SqlCompletionModel(QAbstractItemModel): def last_item(self): """Return the index of the last child (non-category) in the model.""" - for row, table in reversed(list(enumerate(self._categories))): - childcount = table.rowCount() + for row, cat in reversed(list(enumerate(self._categories))): + childcount = cat.rowCount() if childcount > 0: parent = self.index(row, 0) index = self.index(childcount - 1, 0, parent) qtutils.ensure_valid(index) return index return QModelIndex() - - -class SqlException(Exception): - - """Raised on an error interacting with the SQL database.""" - - pass diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index 7c7e3a0c6..3cb46d42d 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -20,17 +20,16 @@ """Functions that return config-related completion models.""" from qutebrowser.config import configdata, configexc -from qutebrowser.completion.models import base +from qutebrowser.completion.models import completionmodel from qutebrowser.utils import objreg def section(): """A CompletionModel filled with settings sections.""" - model = base.CompletionModel(column_widths=(20, 70, 10)) - cat = model.new_category("Sections") - for name in configdata.DATA: - desc = configdata.SECTION_DESC[name].splitlines()[0].strip() - model.new_item(cat, name, desc) + model = completionmodel.CompletionModel(column_widths=(20, 70, 10)) + sections = ((name, configdata.SECTION_DESC[name].splitlines()[0].strip()) + for name in configdata.DATA) + model.add_list("Sections", sections) return model @@ -40,12 +39,12 @@ def option(sectname): Args: sectname: The name of the config section this model shows. """ - model = base.CompletionModel(column_widths=(20, 70, 10)) - cat = model.new_category(sectname) + model = completionmodel.CompletionModel(column_widths=(20, 70, 10)) try: sectdata = configdata.DATA[sectname] except KeyError: return None + options = [] for name in sectdata: try: desc = sectdata.descriptions[name] @@ -57,7 +56,8 @@ def option(sectname): desc = desc.splitlines()[0] config = objreg.get('config') val = config.get(sectname, name, raw=True) - model.new_item(cat, name, desc, val) + options.append((name, desc, val)) + model.add_list(sectname, options) return model @@ -68,16 +68,16 @@ def value(sectname, optname): sectname: The name of the config section this model shows. optname: The name of the config option this model shows. """ - model = base.CompletionModel(column_widths=(20, 70, 10)) - cur_cat = model.new_category("Current/Default") + model = completionmodel.CompletionModel(column_widths=(20, 70, 10)) config = objreg.get('config') + try: - val = config.get(sectname, optname, raw=True) or '""' + current = config.get(sectname, optname, raw=True) or '""' except (configexc.NoSectionError, configexc.NoOptionError): return None - model.new_item(cur_cat, val, "Current value") - default_value = configdata.DATA[sectname][optname].default() or '""' - model.new_item(cur_cat, default_value, "Default value") + + default = configdata.DATA[sectname][optname].default() or '""' + if hasattr(configdata.DATA[sectname], 'valtype'): # Same type for all values (ValueList) vals = configdata.DATA[sectname].valtype.complete() @@ -87,8 +87,9 @@ def value(sectname, optname): "sections, but {} is not!".format(sectname)) # Different type for each value (KeyValue) vals = configdata.DATA[sectname][optname].typ.complete() + + model.add_list("Current/Default", [(current, "Current value"), + (default, "Default value")]) if vals is not None: - cat = model.new_category("Completions") - for (val, desc) in vals: - model.new_item(cat, val, desc) + model.add_list("Completions", vals) return model diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py new file mode 100644 index 000000000..7b222c3c0 --- /dev/null +++ b/qutebrowser/completion/models/listcategory.py @@ -0,0 +1,66 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""The base completion model for completion in the command line. + +Module attributes: + Role: An enum of user defined model roles. +""" + +from PyQt5.QtCore import Qt +from PyQt5.QtGui import QStandardItem, QStandardItemModel + + +class ListCategory(QStandardItemModel): + + """Expose a list of items as a category for the CompletionModel.""" + + def __init__(self, name, items, parent=None): + super().__init__(parent) + self.name = name + # self.setColumnCount(3) TODO needed? + # TODO: batch insert? + # TODO: can I just insert a tuple instead of a list? + for item in items: + self.appendRow([QStandardItem(x) for x in item]) + + def flags(self, index): + """Return the item flags for index. + + Override QAbstractItemModel::flags. + + Args: + index: The QModelIndex to get item flags for. + + Return: + The item flags, or Qt.NoItemFlags on error. + """ + if not index.isValid(): + return + + if index.parent().isValid(): + # item + return (Qt.ItemIsEnabled | Qt.ItemIsSelectable | + Qt.ItemNeverHasChildren) + else: + # category + return Qt.NoItemFlags + + def set_pattern(self, pattern): + pass diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index f5ed72f5d..d07ffaa88 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -22,30 +22,24 @@ from qutebrowser.config import config, configdata from qutebrowser.utils import objreg, log, qtutils from qutebrowser.commands import cmdutils -from qutebrowser.completion.models import base, sqlmodel +from qutebrowser.completion.models import completionmodel def command(): """A CompletionModel filled with non-hidden commands and descriptions.""" - model = base.CompletionModel(column_widths=(20, 60, 20)) + model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_aliases=True, include_hidden=False) - cat = model.new_category("Commands") - for (name, desc, misc) in cmdlist: - model.new_item(cat, name, desc, misc) + model.add_list("Commands", cmdlist) return model def helptopic(): """A CompletionModel filled with help topics.""" - model = base.CompletionModel() + model = completionmodel.CompletionModel() cmdlist = _get_cmd_completions(include_aliases=False, include_hidden=True, prefix=':') - cat = model.new_category("Commands") - for (name, desc, misc) in cmdlist: - model.new_item(cat, name, desc, misc) - - cat = model.new_category("Settings") + settings = [] for sectname, sectdata in configdata.DATA.items(): for optname in sectdata: try: @@ -57,34 +51,35 @@ def helptopic(): else: desc = desc.splitlines()[0] name = '{}->{}'.format(sectname, optname) - model.new_item(cat, name, desc) + settings.append((name, desc)) + + model.add_list("Commands", cmdlist) + model.add_list("Settings", settings) return model def quickmark(): """A CompletionModel filled with all quickmarks.""" - model = base.CompletionModel() - model = sqlmodel.SqlCompletionModel(column_widths=(30, 70, 0)) - model.new_category('Quickmarks') + model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) + model.add_sqltable('Quickmarks') return model def bookmark(): """A CompletionModel filled with all bookmarks.""" - model = base.CompletionModel() - model = sqlmodel.SqlCompletionModel(column_widths=(30, 70, 0)) - model.new_category('Bookmarks') + model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) + model.add_sqltable('Bookmarks') return model def session(): """A CompletionModel filled with session names.""" - model = base.CompletionModel() - cat = model.new_category("Sessions") + model = completionmodel.CompletionModel() try: - for name in objreg.get('session-manager').list_sessions(): - if not name.startswith('_'): - model.new_item(cat, name) + manager = objreg.get('session-manager') + sessions = ((name,) for name in manager.list_sessions() + if not name.startswith('_')) + model.add_list("Sessions", sessions) except OSError: log.completion.exception("Failed to list sessions!") return model @@ -111,7 +106,7 @@ def buffer(): window=int(win_id)) tabbed_browser.on_tab_close_requested(int(tab_index) - 1) - model = base.CompletionModel( + model = completionmodel.CompletionModel( column_widths=(6, 40, 54), delete_cur_item=delete_buffer, columns_to_filter=[idx_column, url_column, text_column]) @@ -121,12 +116,13 @@ def buffer(): window=win_id) if tabbed_browser.shutting_down: continue - c = model.new_category("{}".format(win_id)) + tabs = [] for idx in range(tabbed_browser.count()): tab = tabbed_browser.widget(idx) - model.new_item(c, "{}/{}".format(win_id, idx + 1), - tab.url().toDisplayString(), - tabbed_browser.page_title(idx)) + tabs.append(("{}/{}".format(win_id, idx + 1), + tab.url().toDisplayString(), + tabbed_browser.page_title(idx))) + model.add_list("{}".format(win_id), tabs) return model @@ -137,11 +133,9 @@ def bind(_key): _key: the key being bound. """ # TODO: offer a 'Current binding' completion based on the key. - model = base.CompletionModel(column_widths=(20, 60, 20)) + model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) - cat = model.new_category("Commands") - for (name, desc, misc) in cmdlist: - model.new_item(cat, name, desc, misc) + model.add_list("Commands", cmdlist) return model diff --git a/qutebrowser/completion/models/sortfilter.py b/qutebrowser/completion/models/sortfilter.py index 92d9aeaf1..fb79d002c 100644 --- a/qutebrowser/completion/models/sortfilter.py +++ b/qutebrowser/completion/models/sortfilter.py @@ -41,12 +41,14 @@ class CompletionFilterModel(QSortFilterProxyModel): a long time for some reason. """ - def __init__(self, source, parent=None): + def __init__(self, source, columns_to_filter, parent=None): super().__init__(parent) super().setSourceModel(source) self.srcmodel = source self.pattern = '' self.pattern_re = None + self.columns_to_filter = columns_to_filter + self.name = source.name def set_pattern(self, val): """Setter for pattern. @@ -64,41 +66,6 @@ class CompletionFilterModel(QSortFilterProxyModel): sortcol = 0 self.sort(sortcol) - def count(self): - """Get the count of non-toplevel items currently visible. - - Note this only iterates one level deep, as we only need root items - (categories) and children (items) in our model. - """ - count = 0 - for i in range(self.rowCount()): - cat = self.index(i, 0) - qtutils.ensure_valid(cat) - count += self.rowCount(cat) - return count - - def first_item(self): - """Return the first item in the model.""" - for i in range(self.rowCount()): - cat = self.index(i, 0) - qtutils.ensure_valid(cat) - if cat.model().hasChildren(cat): - index = self.index(0, 0, cat) - qtutils.ensure_valid(index) - return index - return QModelIndex() - - def last_item(self): - """Return the last item in the model.""" - for i in range(self.rowCount() - 1, -1, -1): - cat = self.index(i, 0) - qtutils.ensure_valid(cat) - if cat.model().hasChildren(cat): - index = self.index(self.rowCount(cat) - 1, 0, cat) - qtutils.ensure_valid(index) - return index - return QModelIndex() - def setSourceModel(self, model): """Override QSortFilterProxyModel's setSourceModel to clear pattern.""" log.completion.debug("Setting source model: {}".format(model)) @@ -119,10 +86,10 @@ class CompletionFilterModel(QSortFilterProxyModel): True if self.pattern is contained in item, or if it's a root item (category). False in all other cases """ - if parent == QModelIndex() or not self.pattern: + if not self.pattern: return True - for col in self.srcmodel.columns_to_filter: + for col in self.columns_to_filter: idx = self.srcmodel.index(row, col, parent) if not idx.isValid(): # pragma: no cover # this is a sanity check not hit by any test case diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py new file mode 100644 index 000000000..12e0534c1 --- /dev/null +++ b/qutebrowser/completion/models/sqlcategory.py @@ -0,0 +1,64 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""A completion model backed by SQL tables.""" + +import re + +from PyQt5.QtSql import QSqlQueryModel + +from qutebrowser.misc import sql + + +class SqlCategory(QSqlQueryModel): + """Wraps a SqlQuery for use as a completion category.""" + + def __init__(self, name, *, sort_by, sort_order, select, where, + columns_to_filter, parent=None): + super().__init__(parent=parent) + self.name = name + + query = sql.run_query('select * from {} limit 1'.format(name)) + self._fields = [query.record().fieldName(i) for i in columns_to_filter] + + querystr = 'select {} from {} where ('.format(select, name) + # the incoming pattern will have literal % and _ escaped with '\' + # we need to tell sql to treat '\' as an escape character + querystr += ' or '.join("{} like ? escape '\\'".format(f) + for f in self._fields) + querystr += ')' + if where: + querystr += ' and ' + where + + if sort_by: + assert sort_order == 'asc' or sort_order == 'desc' + querystr += ' order by {} {}'.format(sort_by, sort_order) + + self._querystr = querystr + self.set_pattern('') + + def set_pattern(self, pattern): + # escape to treat a user input % or _ as a literal, not a wildcard + pattern = pattern.replace('%', '\\%') + pattern = pattern.replace('_', '\\_') + # treat spaces as wildcards to match any of the typed words + pattern = re.sub(r' +', '%', pattern) + pattern = '%{}%'.format(pattern) + query = sql.run_query(self._querystr, [pattern for _ in self._fields]) + self.setQuery(query) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 0b357c3d7..a38e107d2 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -19,7 +19,7 @@ """Function to return the url completion model for the `open` command.""" -from qutebrowser.completion.models import sqlmodel +from qutebrowser.completion.models import completionmodel from qutebrowser.config import config from qutebrowser.utils import qtutils, log, objreg @@ -61,14 +61,16 @@ def url(): Used for the `open` command. """ - model = sqlmodel.SqlCompletionModel(column_widths=(40, 50, 10), - columns_to_filter=[_URLCOL, _TEXTCOL], - delete_cur_item=_delete_url) + model = completionmodel.CompletionModel( + column_widths=(40, 50, 10), + columns_to_filter=[_URLCOL, _TEXTCOL], + delete_cur_item=_delete_url) + timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) - model.new_category('Quickmarks', select='url, name') - model.new_category('Bookmarks') - model.new_category('History', + model.add_sqltable('Quickmarks', select='url, name') + model.add_sqltable('Bookmarks') + model.add_sqltable('History', sort_order='desc', sort_by='atime', select='url, title, {}'.format(select_time), where='not redirect') diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py new file mode 100644 index 000000000..4c1241d12 --- /dev/null +++ b/tests/unit/completion/test_completionmodel.py @@ -0,0 +1,146 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Tests for CompletionModel.""" + +import pytest + +from qutebrowser.completion.models import completionmodel, sortfilter + + +def _create_model(data, filter_cols=None): + """Create a completion model populated with the given data. + + data: A list of lists, where each sub-list represents a category, each + tuple in the sub-list represents an item, and each value in the + tuple represents the item data for that column + filter_cols: Columns to filter, or None for default. + """ + model = completionmodel.CompletionModel(columns_to_filter=filter_cols) + for catdata in data: + model.add_list('', catdata) + return model + + +def _extract_model_data(model): + """Express a model's data as a list for easier comparison. + + Return: A list of lists, where each sub-list represents a category, each + tuple in the sub-list represents an item, and each value in the + tuple represents the item data for that column + """ + data = [] + for i in range(0, model.rowCount()): + cat_idx = model.index(i, 0) + row = [] + for j in range(0, model.rowCount(cat_idx)): + row.append((model.data(cat_idx.child(j, 0)), + model.data(cat_idx.child(j, 1)), + model.data(cat_idx.child(j, 2)))) + data.append(row) + return data + + +@pytest.mark.parametrize('tree, first, last', [ + ([[('Aa',)]], 'Aa', 'Aa'), + ([[('Aa',)], [('Ba',)]], 'Aa', 'Ba'), + ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], + 'Aa', 'Ca'), + ([[], [('Ba',)]], 'Ba', 'Ba'), + ([[], [], [('Ca',)]], 'Ca', 'Ca'), + ([[], [], [('Ca',), ('Cb',)]], 'Ca', 'Cb'), + ([[('Aa',)], []], 'Aa', 'Aa'), + ([[('Aa',)], []], 'Aa', 'Aa'), + ([[('Aa',)], [], []], 'Aa', 'Aa'), + ([[('Aa',)], [], [('Ca',)]], 'Aa', 'Ca'), + ([[], []], None, None), +]) +def test_first_last_item(tree, first, last): + """Test that first() and last() return indexes to the first and last items. + + Args: + tree: Each list represents a completion category, with each string + being an item under that category. + first: text of the first item + last: text of the last item + """ + model = _create_model(tree) + assert model.data(model.first_item()) == first + assert model.data(model.last_item()) == last + + +@pytest.mark.parametrize('tree, expected', [ + ([[('Aa',)]], 1), + ([[('Aa',)], [('Ba',)]], 2), + ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], 6), + ([[], [('Ba',)]], 1), + ([[], [], [('Ca',)]], 1), + ([[], [], [('Ca',), ('Cb',)]], 2), + ([[('Aa',)], []], 1), + ([[('Aa',)], []], 1), + ([[('Aa',)], [], []], 1), + ([[('Aa',)], [], [('Ca',)]], 2), +]) +def test_count(tree, expected): + model = _create_model(tree) + assert model.count() == expected + + +@pytest.mark.parametrize('pattern, filter_cols, before, after', [ + ('foo', [0], + [[('foo', '', ''), ('bar', '', '')]], + [[('foo', '', '')]]), + + ('foo', [0], + [[('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')]], + [[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]]), + + ('foo', [0], + [[('foo', '', '')], [('bar', '', '')]], + [[('foo', '', '')], []]), + + # prefer foobar as it starts with the pattern + ('foo', [0], + [[('barfoo', '', ''), ('foobar', '', '')]], + [[('foobar', '', ''), ('barfoo', '', '')]]), + + # however, don't rearrange categories + ('foo', [0], + [[('barfoo', '', '')], [('foobar', '', '')]], + [[('barfoo', '', '')], [('foobar', '', '')]]), + + ('foo', [1], + [[('foo', 'bar', ''), ('bar', 'foo', '')]], + [[('bar', 'foo', '')]]), + + ('foo', [0, 1], + [[('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')]], + [[('foo', 'bar', ''), ('bar', 'foo', '')]]), + + ('foo', [0, 1, 2], + [[('foo', '', ''), ('bar', '')]], + [[('foo', '', '')]]), +]) +def test_set_pattern(pattern, filter_cols, before, after): + """Validate the filtering and sorting results of set_pattern.""" + # TODO: just test that it calls the mock on its child categories + model = _create_model(before, filter_cols) + model.set_pattern(pattern) + actual = _extract_model_data(model) + assert actual == after diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 2c8d96713..3db4dca5a 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -25,7 +25,7 @@ import pytest from PyQt5.QtGui import QStandardItem, QColor from qutebrowser.completion import completionwidget -from qutebrowser.completion.models import base, sortfilter +from qutebrowser.completion.models import completionmodel from qutebrowser.commands import cmdexc @@ -72,28 +72,25 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry, @pytest.fixture def simplemodel(completionview): - """A filter model wrapped around a completion model with one item.""" - model = base.CompletionModel() - cat = QStandardItem() - cat.appendRow(QStandardItem('foo')) - model.appendRow(cat) - return sortfilter.CompletionFilterModel(model, parent=completionview) + """A completion model with one item.""" + model = completionmodel.CompletionModel() + model.add_list('', [('foo'),]) + return model def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" - model = base.CompletionModel() - filtermodel = sortfilter.CompletionFilterModel(model) + model = completionmodel.CompletionModel() for i in range(3): - model.appendRow(QStandardItem(str(i))) - completionview.set_model(filtermodel) - assert completionview.model() is filtermodel + model.add_list(str(i), [('foo',)]) + completionview.set_model(model) + assert completionview.model() is model for i in range(model.rowCount()): - assert completionview.isExpanded(filtermodel.index(i, 0)) + assert completionview.isExpanded(model.index(i, 0)) def test_set_pattern(completionview): - model = sortfilter.CompletionFilterModel(base.CompletionModel()) + model = completionmodel.CompletionModel() model.set_pattern = unittest.mock.Mock() completionview.set_model(model, 'foo') model.set_pattern.assert_called_with('foo') @@ -159,15 +156,10 @@ def test_completion_item_focus(which, tree, expected, completionview, qtbot): successive movement. None implies no signal should be emitted. """ - model = base.CompletionModel() + model = completionmodel.CompletionModel() for catdata in tree: - cat = QStandardItem() - model.appendRow(cat) - for name in catdata: - cat.appendRow(QStandardItem(name)) - filtermodel = sortfilter.CompletionFilterModel(model, - parent=completionview) - completionview.set_model(filtermodel) + model.add_list('', (x,) for x in catdata) + completionview.set_model(model) for entry in expected: if entry is None: with qtbot.assertNotEmitted(completionview.selection_changed): @@ -187,10 +179,8 @@ def test_completion_item_focus_no_model(which, completionview, qtbot): """ with qtbot.assertNotEmitted(completionview.selection_changed): completionview.completion_item_focus(which) - model = base.CompletionModel() - filtermodel = sortfilter.CompletionFilterModel(model, - parent=completionview) - completionview.set_model(filtermodel) + model = completionmodel.CompletionModel() + completionview.set_model(model) completionview.set_model(None) with qtbot.assertNotEmitted(completionview.selection_changed): completionview.completion_item_focus(which) @@ -211,16 +201,12 @@ def test_completion_show(show, rows, quick_complete, completionview, config_stub.data['completion']['show'] = show config_stub.data['completion']['quick-complete'] = quick_complete - model = base.CompletionModel() + model = completionmodel.CompletionModel() for name in rows: - cat = QStandardItem() - model.appendRow(cat) - cat.appendRow(QStandardItem(name)) - filtermodel = sortfilter.CompletionFilterModel(model, - parent=completionview) + model.add_list('', [(name,)]) assert not completionview.isVisible() - completionview.set_model(filtermodel) + completionview.set_model(model) assert completionview.isVisible() == (show == 'always' and len(rows) > 0) completionview.completion_item_focus('next') expected = (show != 'never' and len(rows) > 0 and diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 224d7ed59..6d68cc8ab 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -228,12 +228,12 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): (':hide', '', ''), ], "Settings": [ - ('general->time', 'Is an illusion.', ''), - ('general->volume', 'Goes to 11', ''), - ('ui->gesture', 'Waggle your hands to control qutebrowser', ''), - ('ui->mind', 'Enable mind-control ui (experimental)', ''), - ('ui->voice', 'Whether to respond to voice commands', ''), - ('searchengines->DEFAULT', '', ''), + ('general->time', 'Is an illusion.', None), + ('general->volume', 'Goes to 11', None), + ('ui->gesture', 'Waggle your hands to control qutebrowser', None), + ('ui->mind', 'Enable mind-control ui (experimental)', None), + ('ui->voice', 'Whether to respond to voice commands', None), + ('searchengines->DEFAULT', '', None), ] }) @@ -342,7 +342,9 @@ def test_session_completion(qtmodeltester, session_manager_stub): qtmodeltester.check(model) _check_completions(model, { - "Sessions": [('default', '', ''), ('1', '', ''), ('2', '', '')] + "Sessions": [('default', None, None), + ('1', None, None), + ('2', None, None)] }) @@ -406,9 +408,9 @@ def test_setting_section_completion(qtmodeltester, monkeypatch, stubs): _check_completions(model, { "Sections": [ - ('general', 'General/miscellaneous options.', ''), - ('ui', 'General options related to the user interface.', ''), - ('searchengines', 'Definitions of search engines ...', ''), + ('general', 'General/miscellaneous options.', None), + ('ui', 'General options related to the user interface.', None), + ('searchengines', 'Definitions of search engines ...', None), ] }) @@ -462,12 +464,12 @@ def test_setting_value_completion(qtmodeltester, monkeypatch, stubs, _check_completions(model, { "Current/Default": [ - ('0', 'Current value', ''), - ('11', 'Default value', ''), + ('0', 'Current value', None), + ('11', 'Default value', None), ], "Completions": [ - ('0', '', ''), - ('11', '', ''), + ('0', '', None), + ('11', '', None), ] }) diff --git a/tests/unit/completion/test_sortfilter.py b/tests/unit/completion/test_sortfilter.py index 92d644810..7dadaa9dd 100644 --- a/tests/unit/completion/test_sortfilter.py +++ b/tests/unit/completion/test_sortfilter.py @@ -21,9 +21,10 @@ import pytest -from qutebrowser.completion.models import base, sortfilter +from qutebrowser.completion.models import listcategory, sortfilter +# TODO: merge listcategory and sortfilter def _create_model(data): """Create a completion model populated with the given data. @@ -31,11 +32,9 @@ def _create_model(data): tuple in the sub-list represents an item, and each value in the tuple represents the item data for that column """ - model = base.CompletionModel() + model = completionmodel.CompletionModel() for catdata in data: - cat = model.new_category('') - for itemdata in catdata: - model.new_item(cat, *itemdata) + cat = model.add_list(itemdata) return model @@ -72,7 +71,7 @@ def _extract_model_data(model): ('4', 'blah', False), ]) def test_filter_accepts_row(pattern, data, expected): - source_model = base.CompletionModel() + source_model = completionmodel.CompletionModel() cat = source_model.new_category('test') source_model.new_item(cat, data) @@ -86,35 +85,6 @@ def test_filter_accepts_row(pattern, data, expected): assert row_count == (1 if expected else 0) -@pytest.mark.parametrize('tree, first, last', [ - ([[('Aa',)]], 'Aa', 'Aa'), - ([[('Aa',)], [('Ba',)]], 'Aa', 'Ba'), - ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], - 'Aa', 'Ca'), - ([[], [('Ba',)]], 'Ba', 'Ba'), - ([[], [], [('Ca',)]], 'Ca', 'Ca'), - ([[], [], [('Ca',), ('Cb',)]], 'Ca', 'Cb'), - ([[('Aa',)], []], 'Aa', 'Aa'), - ([[('Aa',)], []], 'Aa', 'Aa'), - ([[('Aa',)], [], []], 'Aa', 'Aa'), - ([[('Aa',)], [], [('Ca',)]], 'Aa', 'Ca'), - ([[], []], None, None), -]) -def test_first_last_item(tree, first, last): - """Test that first() and last() return indexes to the first and last items. - - Args: - tree: Each list represents a completion category, with each string - being an item under that category. - first: text of the first item - last: text of the last item - """ - model = _create_model(tree) - filter_model = sortfilter.CompletionFilterModel(model) - assert filter_model.data(filter_model.first_item()) == first - assert filter_model.data(filter_model.last_item()) == last - - def test_set_source_model(): """Ensure setSourceModel sets source_model and clears the pattern.""" model1 = base.CompletionModel() @@ -131,24 +101,6 @@ def test_set_source_model(): assert not filter_model.pattern -@pytest.mark.parametrize('tree, expected', [ - ([[('Aa',)]], 1), - ([[('Aa',)], [('Ba',)]], 2), - ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], 6), - ([[], [('Ba',)]], 1), - ([[], [], [('Ca',)]], 1), - ([[], [], [('Ca',), ('Cb',)]], 2), - ([[('Aa',)], []], 1), - ([[('Aa',)], []], 1), - ([[('Aa',)], [], []], 1), - ([[('Aa',)], [], [('Ca',)]], 2), -]) -def test_count(tree, expected): - model = _create_model(tree) - filter_model = sortfilter.CompletionFilterModel(model) - assert filter_model.count() == expected - - @pytest.mark.parametrize('pattern, filter_cols, before, after', [ ('foo', [0], [[('foo', '', ''), ('bar', '', '')]], From 80619c88b39cdb16c86a096da2fbc0483185976b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 28 Feb 2017 08:43:11 -0500 Subject: [PATCH 048/337] Revert "Use SQL for quickmark/bookmark storage." This reverts commit fa1ebb03b70dfff4ac64038e67d9bab04b984de5. --- qutebrowser/browser/commands.py | 12 ++-- qutebrowser/browser/urlmarks.py | 110 ++++++++++++++++++++++---------- 2 files changed, 81 insertions(+), 41 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index e5ab0af21..8914d4ac2 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1260,15 +1260,13 @@ class CommandDispatcher: if name is None: url = self._current_url() try: - quickmark_manager.delete_by_qurl(url) + name = quickmark_manager.get_by_qurl(url) except urlmarks.DoesNotExistError as e: raise cmdexc.CommandError(str(e)) - else: - try: - quickmark_manager.delete(name) - except KeyError: - raise cmdexc.CommandError( - "Quickmark '{}' not found!".format(name)) + try: + quickmark_manager.delete(name) + except KeyError: + raise cmdexc.CommandError("Quickmark '{}' not found!".format(name)) @cmdutils.register(instance='command-dispatcher', scope='window') def bookmark_add(self, url=None, title=None, toggle=False): diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index 580d77881..013de408c 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -29,13 +29,14 @@ import os import html import os.path import functools +import collections -from PyQt5.QtCore import QUrl +from PyQt5.QtCore import pyqtSignal, QUrl, QObject from qutebrowser.utils import (message, usertypes, qtutils, urlutils, standarddir, objreg, log) from qutebrowser.commands import cmdutils -from qutebrowser.misc import lineparser, sql +from qutebrowser.misc import lineparser class Error(Exception): @@ -66,18 +67,29 @@ class AlreadyExistsError(Error): pass -class UrlMarkManager(sql.SqlTable): +class UrlMarkManager(QObject): """Base class for BookmarkManager and QuickmarkManager. Attributes: marks: An OrderedDict of all quickmarks/bookmarks. _lineparser: The LineParser used for the marks + + Signals: + changed: Emitted when anything changed. + added: Emitted when a new quickmark/bookmark was added. + removed: Emitted when an existing quickmark/bookmark was removed. """ - def __init__(self, name, fields, primary_key, parent=None): - """Initialize and read marks.""" - super().__init__(name, fields, primary_key, parent) + changed = pyqtSignal() + added = pyqtSignal(str, str) + removed = pyqtSignal(str) + + def __init__(self, parent=None): + """Initialize and read quickmarks.""" + super().__init__(parent) + + self.marks = collections.OrderedDict() self._init_lineparser() for line in self._lineparser: @@ -98,17 +110,31 @@ class UrlMarkManager(sql.SqlTable): def save(self): """Save the marks to disk.""" - self._lineparser.data = [' '.join(tpl) for tpl in self] + self._lineparser.data = [' '.join(tpl) for tpl in self.marks.items()] self._lineparser.save() + def delete(self, key): + """Delete a quickmark/bookmark. + + Args: + key: The key to delete (name for quickmarks, URL for bookmarks.) + """ + del self.marks[key] + self.changed.emit() + self.removed.emit(key) + class QuickmarkManager(UrlMarkManager): - """Manager for quickmarks.""" + """Manager for quickmarks. - def __init__(self, parent=None): - super().__init__('Quickmarks', ['name', 'url'], primary_key='name', - parent=parent) + The primary key for quickmarks is their *name*, this means: + + - self.marks maps names to URLs. + - changed gets emitted with the name as first argument and the URL as + second argument. + - removed gets emitted with the name as argument. + """ def _init_lineparser(self): self._lineparser = lineparser.LineParser( @@ -125,7 +151,7 @@ class QuickmarkManager(UrlMarkManager): except ValueError: message.error("Invalid quickmark '{}'".format(line)) else: - self.insert([key, url]) + self.marks[key] = url def prompt_save(self, url): """Prompt for a new quickmark name to be added and add it. @@ -165,32 +191,41 @@ class QuickmarkManager(UrlMarkManager): def set_mark(): """Really set the quickmark.""" - self.insert([name, url], replace=True) + self.marks[name] = url + self.changed.emit() + self.added.emit(name, url) log.misc.debug("Added quickmark {} for {}".format(name, url)) - if name in self: + if name in self.marks: message.confirm_async( title="Override existing quickmark?", yes_action=set_mark, default=True) else: set_mark() - def delete_by_qurl(self, url): - """Delete a quickmark by QUrl (as opposed to by name).""" + def get_by_qurl(self, url): + """Look up a quickmark by QUrl, returning its name. + + Takes O(n) time, where n is the number of quickmarks. + Use a name instead where possible. + """ qtutils.ensure_valid(url) urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) + try: - self.delete(urlstr, field='url') - except KeyError: - raise DoesNotExistError("Quickmark for '{}' not found!" - .format(urlstr)) + index = list(self.marks.values()).index(urlstr) + key = list(self.marks.keys())[index] + except ValueError: + raise DoesNotExistError( + "Quickmark for '{}' not found!".format(urlstr)) + return key def get(self, name): """Get the URL of the quickmark named name as a QUrl.""" - if name not in self: - raise DoesNotExistError("Quickmark '{}' does not exist!" - .format(name)) - urlstr = self[name].url + if name not in self.marks: + raise DoesNotExistError( + "Quickmark '{}' does not exist!".format(name)) + urlstr = self.marks[name] try: url = urlutils.fuzzy_url(urlstr, do_search=False) except urlutils.InvalidUrlError as e: @@ -201,11 +236,15 @@ class QuickmarkManager(UrlMarkManager): class BookmarkManager(UrlMarkManager): - """Manager for bookmarks.""" + """Manager for bookmarks. - def __init__(self, parent=None): - super().__init__('Bookmarks', ['url', 'title'], primary_key='url', - parent=parent) + The primary key for bookmarks is their *url*, this means: + + - self.marks maps URLs to titles. + - changed gets emitted with the URL as first argument and the title as + second argument. + - removed gets emitted with the URL as argument. + """ def _init_lineparser(self): bookmarks_directory = os.path.join(standarddir.config(), 'bookmarks') @@ -223,9 +262,10 @@ class BookmarkManager(UrlMarkManager): def _parse_line(self, line): parts = line.split(maxsplit=1) - urlstr = parts[0] - title = parts[1] if len(parts) == 2 else '' - self.insert([urlstr, title]) + if len(parts) == 2: + self.marks[parts[0]] = parts[1] + elif len(parts) == 1: + self.marks[parts[0]] = '' def add(self, url, title, *, toggle=False): """Add a new bookmark. @@ -246,12 +286,14 @@ class BookmarkManager(UrlMarkManager): urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) - if urlstr in self: + if urlstr in self.marks: if toggle: - self.delete(urlstr) + del self.marks[urlstr] return False else: raise AlreadyExistsError("Bookmark already exists!") else: - self.insert([urlstr, title]) + self.marks[urlstr] = title + self.changed.emit() + self.added.emit(title, urlstr) return True From 1d54688b0b3e74d59d18b9b4dc0f6509a96a4018 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 28 Feb 2017 08:44:17 -0500 Subject: [PATCH 049/337] Revert "Use SQL completer for quickmarks/bookmarks." This reverts commit bcf1520132df84552f69419f3b1cbf3ede20ccad. --- tests/unit/completion/test_models.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 6d68cc8ab..6e7e60958 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -238,6 +238,7 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): }) +@pytest.mark.skip def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() @@ -246,13 +247,14 @@ def test_quickmark_completion(qtmodeltester, quickmarks): _check_completions(model, { "Quickmarks": [ - ('aw', 'https://wiki.archlinux.org', None), - ('ddg', 'https://duckduckgo.com', None), - ('wiki', 'https://wikipedia.org', None), + ('aw', 'https://wiki.archlinux.org', ''), + ('ddg', 'https://duckduckgo.com', ''), + ('wiki', 'https://wikipedia.org', ''), ] }) +@pytest.mark.skip def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() @@ -261,9 +263,9 @@ def test_bookmark_completion(qtmodeltester, bookmarks): _check_completions(model, { "Bookmarks": [ - ('https://github.com', 'GitHub', None), - ('https://python.org', 'Welcome to Python.org', None), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), + ('https://github.com', 'GitHub', ''), + ('https://python.org', 'Welcome to Python.org', ''), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', ''), ] }) From ce3c5557123ee074db3a9092400f71f3beb38051 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 28 Feb 2017 08:50:55 -0500 Subject: [PATCH 050/337] Use list completion for bookmarks/quickmarks. The RFC on moving from plaintext to SQL storage (#2340) showed that many would be upset if bookmarks and quickmarks were no longer stored in plaintext. This commit uses list-based completion for quickmarks and bookmarks. Now the history storage can be moved from plaintext to an on-disk SQL database while leaving bookmarks and quickmarks as-is. --- qutebrowser/completion/models/miscmodels.py | 4 ++-- qutebrowser/completion/models/urlmodel.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index d07ffaa88..64a467a6b 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -61,14 +61,14 @@ def helptopic(): def quickmark(): """A CompletionModel filled with all quickmarks.""" model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) - model.add_sqltable('Quickmarks') + model.add_list('Quickmarks', objreg.get('quickmark-manager').marks.items()) return model def bookmark(): """A CompletionModel filled with all bookmarks.""" model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) - model.add_sqltable('Bookmarks') + model.add_list('Bookmarks', objreg.get('bookmark-manager').marks.items()) return model diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index a38e107d2..5b738c503 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -66,10 +66,13 @@ def url(): columns_to_filter=[_URLCOL, _TEXTCOL], delete_cur_item=_delete_url) + quickmarks = objreg.get('quickmark-manager').marks.items() + model.add_list('Quickmarks', ((url, name) for (name, url) in quickmarks)) + + model.add_list('Bookmarks', objreg.get('bookmark-manager').marks.items()) + timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) - model.add_sqltable('Quickmarks', select='url, name') - model.add_sqltable('Bookmarks') model.add_sqltable('History', sort_order='desc', sort_by='atime', select='url, title, {}'.format(select_time), From f95dff4d9e39abc3c29c514a3783ce632f96c369 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 3 Mar 2017 08:31:28 -0500 Subject: [PATCH 051/337] Decouple categories from completionmodel. Instead of add_list and add_sqltable, the completion model now supports add_category, and callees either pass in a SqlCategory or ListCategory. This makes unit testing much easier. This also folds CompletionFilterModel into the ListCategory class. --- qutebrowser/browser/commands.py | 9 +- qutebrowser/completion/completionwidget.py | 1 + .../completion/models/completionmodel.py | 32 +-- qutebrowser/completion/models/configmodel.py | 13 +- qutebrowser/completion/models/listcategory.py | 102 ++++++-- qutebrowser/completion/models/miscmodels.py | 21 +- qutebrowser/completion/models/sortfilter.py | 133 ---------- qutebrowser/completion/models/sqlcategory.py | 44 ++-- qutebrowser/completion/models/urlmodel.py | 22 +- scripts/dev/check_coverage.py | 2 - tests/helpers/fixtures.py | 18 ++ tests/unit/completion/test_completer.py | 3 +- tests/unit/completion/test_completionmodel.py | 163 ++++--------- .../unit/completion/test_completionwidget.py | 48 ++-- tests/unit/completion/test_listcategory.py | 70 ++++++ tests/unit/completion/test_models.py | 46 ++-- tests/unit/completion/test_sortfilter.py | 146 ----------- tests/unit/completion/test_sqlcateogry.py | 156 ++++++++++++ tests/unit/completion/test_sqlmodel.py | 230 ------------------ 19 files changed, 484 insertions(+), 775 deletions(-) delete mode 100644 qutebrowser/completion/models/sortfilter.py create mode 100644 tests/unit/completion/test_listcategory.py delete mode 100644 tests/unit/completion/test_sortfilter.py create mode 100644 tests/unit/completion/test_sqlcateogry.py delete mode 100644 tests/unit/completion/test_sqlmodel.py diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 8914d4ac2..3ec4c63c0 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -41,7 +41,7 @@ from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils, objreg, utils, typing) from qutebrowser.utils.usertypes import KeyMode from qutebrowser.misc import editor, guiprocess -from qutebrowser.completion.models import sortfilter, urlmodel, miscmodels +from qutebrowser.completion.models import urlmodel, miscmodels class CommandDispatcher: @@ -1024,10 +1024,9 @@ class CommandDispatcher: int(part) except ValueError: model = miscmodels.buffer() - sf = sortfilter.CompletionFilterModel(source=model) - sf.set_pattern(index) - if sf.count() > 0: - index = sf.data(sf.first_item()) + model.set_pattern(index) + if model.count() > 0: + index = model.data(model.first_item()) index_parts = index.split('/', 1) else: raise cmdexc.CommandError( diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index f6a980612..5e207d68d 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -275,6 +275,7 @@ class CompletionView(QTreeView): self.hide() return + model.setParent(self) old_model = self.model() if model is not old_model: sel_model = self.selectionModel() diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index b8dcc4601..84dd74b8f 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -24,7 +24,6 @@ import re from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from qutebrowser.utils import log, qtutils -from qutebrowser.completion.models import sortfilter, listcategory, sqlcategory class CompletionModel(QAbstractItemModel): @@ -65,33 +64,8 @@ class CompletionModel(QAbstractItemModel): return self._categories[index.row()] return None - def add_list(self, name, items): - """Add a list of items as a completion category. - - Args: - name: Title of the category. - items: List of tuples. - """ - cat = listcategory.ListCategory(name, items, parent=self) - filtermodel = sortfilter.CompletionFilterModel(cat, - self.columns_to_filter) - self._categories.append(filtermodel) - - def add_sqltable(self, name, *, select='*', where=None, sort_by=None, - sort_order=None): - """Create a new completion category and add it to this model. - - Args: - name: Name of category, and the table in the database. - select: A custom result column expression for the select statement. - where: An optional clause to filter out some rows. - sort_by: The name of the field to sort by, or None for no sorting. - sort_order: Either 'asc' or 'desc', if sort_by is non-None - """ - cat = sqlcategory.SqlCategory(name, parent=self, sort_by=sort_by, - sort_order=sort_order, - select=select, where=where, - columns_to_filter=self.columns_to_filter) + def add_category(self, cat): + """Add a completion category to the model.""" self._categories.append(cat) def data(self, index, role=Qt.DisplayRole): @@ -210,7 +184,7 @@ class CompletionModel(QAbstractItemModel): # TODO: should pattern be saved in the view layer instead? self.pattern = pattern for cat in self._categories: - cat.set_pattern(pattern) + cat.set_pattern(pattern, self.columns_to_filter) def first_item(self): """Return the index of the first child (non-category) in the model.""" diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index 3cb46d42d..663a0b7f7 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -20,7 +20,7 @@ """Functions that return config-related completion models.""" from qutebrowser.config import configdata, configexc -from qutebrowser.completion.models import completionmodel +from qutebrowser.completion.models import completionmodel, listcategory from qutebrowser.utils import objreg @@ -29,7 +29,7 @@ def section(): model = completionmodel.CompletionModel(column_widths=(20, 70, 10)) sections = ((name, configdata.SECTION_DESC[name].splitlines()[0].strip()) for name in configdata.DATA) - model.add_list("Sections", sections) + model.add_category(listcategory.ListCategory("Sections", sections)) return model @@ -57,7 +57,7 @@ def option(sectname): config = objreg.get('config') val = config.get(sectname, name, raw=True) options.append((name, desc, val)) - model.add_list(sectname, options) + model.add_category(listcategory.ListCategory(sectname, options)) return model @@ -88,8 +88,9 @@ def value(sectname, optname): # Different type for each value (KeyValue) vals = configdata.DATA[sectname][optname].typ.complete() - model.add_list("Current/Default", [(current, "Current value"), - (default, "Default value")]) + cur_cat = listcategory.ListCategory("Current/Default", + [(current, "Current value"), (default, "Default value")]) + model.add_category(cur_cat) if vals is not None: - model.add_list("Completions", vals) + model.add_category(listcategory.ListCategory("Completions", vals)) return model diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index 7b222c3c0..6251a7cac 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -23,44 +23,100 @@ Module attributes: Role: An enum of user defined model roles. """ -from PyQt5.QtCore import Qt +import re + +from PyQt5.QtCore import Qt, QSortFilterProxyModel, QModelIndex from PyQt5.QtGui import QStandardItem, QStandardItemModel +from qutebrowser.utils import qtutils, debug, log -class ListCategory(QStandardItemModel): + +class ListCategory(QSortFilterProxyModel): """Expose a list of items as a category for the CompletionModel.""" def __init__(self, name, items, parent=None): super().__init__(parent) self.name = name - # self.setColumnCount(3) TODO needed? - # TODO: batch insert? - # TODO: can I just insert a tuple instead of a list? + self.srcmodel = QStandardItemModel(parent=self) + self.pattern = '' + self.pattern_re = None for item in items: - self.appendRow([QStandardItem(x) for x in item]) + self.srcmodel.appendRow([QStandardItem(x) for x in item]) + self.setSourceModel(self.srcmodel) - def flags(self, index): - """Return the item flags for index. - - Override QAbstractItemModel::flags. + def set_pattern(self, val, columns_to_filter): + """Setter for pattern. Args: - index: The QModelIndex to get item flags for. + val: The value to set. + """ + with debug.log_time(log.completion, 'Setting filter pattern'): + self.columns_to_filter = columns_to_filter + self.pattern = val + val = re.sub(r' +', r' ', val) # See #1919 + val = re.escape(val) + val = val.replace(r'\ ', '.*') + self.pattern_re = re.compile(val, re.IGNORECASE) + self.invalidate() + sortcol = 0 + self.sort(sortcol) + + def filterAcceptsRow(self, row, parent): + """Custom filter implementation. + + Override QSortFilterProxyModel::filterAcceptsRow. + + Args: + row: The row of the item. + parent: The parent item QModelIndex. Return: - The item flags, or Qt.NoItemFlags on error. + True if self.pattern is contained in item, or if it's a root item + (category). False in all other cases """ - if not index.isValid(): - return + if not self.pattern: + return True - if index.parent().isValid(): - # item - return (Qt.ItemIsEnabled | Qt.ItemIsSelectable | - Qt.ItemNeverHasChildren) + for col in self.columns_to_filter: + idx = self.srcmodel.index(row, col, parent) + if not idx.isValid(): # pragma: no cover + # this is a sanity check not hit by any test case + continue + data = self.srcmodel.data(idx) + if not data: + continue + elif self.pattern_re.search(data): + return True + return False + + def lessThan(self, lindex, rindex): + """Custom sorting implementation. + + Prefers all items which start with self.pattern. Other than that, uses + normal Python string sorting. + + Args: + lindex: The QModelIndex of the left item (*left* < right) + rindex: The QModelIndex of the right item (left < *right*) + + Return: + True if left < right, else False + """ + qtutils.ensure_valid(lindex) + qtutils.ensure_valid(rindex) + + left = self.srcmodel.data(lindex) + right = self.srcmodel.data(rindex) + + leftstart = left.startswith(self.pattern) + rightstart = right.startswith(self.pattern) + + if leftstart and rightstart: + return left < right + elif leftstart: + return True + elif rightstart: + return False else: - # category - return Qt.NoItemFlags - - def set_pattern(self, pattern): - pass + return left < right diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 64a467a6b..368e1e6c6 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -22,14 +22,14 @@ from qutebrowser.config import config, configdata from qutebrowser.utils import objreg, log, qtutils from qutebrowser.commands import cmdutils -from qutebrowser.completion.models import completionmodel +from qutebrowser.completion.models import completionmodel, listcategory def command(): """A CompletionModel filled with non-hidden commands and descriptions.""" model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_aliases=True, include_hidden=False) - model.add_list("Commands", cmdlist) + model.add_category(listcategory.ListCategory("Commands", cmdlist)) return model @@ -53,22 +53,24 @@ def helptopic(): name = '{}->{}'.format(sectname, optname) settings.append((name, desc)) - model.add_list("Commands", cmdlist) - model.add_list("Settings", settings) + model.add_category(listcategory.ListCategory("Commands", cmdlist)) + model.add_category(listcategory.ListCategory("Settings", settings)) return model def quickmark(): """A CompletionModel filled with all quickmarks.""" model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) - model.add_list('Quickmarks', objreg.get('quickmark-manager').marks.items()) + marks = objreg.get('quickmark-manager').marks.items() + model.add_category(listcategory.ListCategory('Quickmarks', marks)) return model def bookmark(): """A CompletionModel filled with all bookmarks.""" model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) - model.add_list('Bookmarks', objreg.get('bookmark-manager').marks.items()) + marks = objreg.get('bookmark-manager').marks.items() + model.add_category(listcategory.ListCategory('Bookmarks', marks)) return model @@ -79,7 +81,7 @@ def session(): manager = objreg.get('session-manager') sessions = ((name,) for name in manager.list_sessions() if not name.startswith('_')) - model.add_list("Sessions", sessions) + model.add_category(listcategory.ListCategory("Sessions", sessions)) except OSError: log.completion.exception("Failed to list sessions!") return model @@ -122,7 +124,8 @@ def buffer(): tabs.append(("{}/{}".format(win_id, idx + 1), tab.url().toDisplayString(), tabbed_browser.page_title(idx))) - model.add_list("{}".format(win_id), tabs) + cat = listcategory.ListCategory("{}".format(win_id), tabs) + model.add_category(cat) return model @@ -135,7 +138,7 @@ def bind(_key): # TODO: offer a 'Current binding' completion based on the key. model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) - model.add_list("Commands", cmdlist) + model.add_category(listcategory.ListCategory("Commands", cmdlist)) return model diff --git a/qutebrowser/completion/models/sortfilter.py b/qutebrowser/completion/models/sortfilter.py deleted file mode 100644 index fb79d002c..000000000 --- a/qutebrowser/completion/models/sortfilter.py +++ /dev/null @@ -1,133 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2014-2017 Florian Bruhin (The Compiler) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""A filtering/sorting base model for completions. - -Contains: - CompletionFilterModel -- A QSortFilterProxyModel subclass for completions. -""" - -import re - -from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex - -from qutebrowser.utils import log, qtutils, debug - - -class CompletionFilterModel(QSortFilterProxyModel): - - """Subclass of QSortFilterProxyModel with custom filtering. - - Attributes: - pattern: The pattern to filter with. - srcmodel: The current source model. - Kept as attribute because calling `sourceModel` takes quite - a long time for some reason. - """ - - def __init__(self, source, columns_to_filter, parent=None): - super().__init__(parent) - super().setSourceModel(source) - self.srcmodel = source - self.pattern = '' - self.pattern_re = None - self.columns_to_filter = columns_to_filter - self.name = source.name - - def set_pattern(self, val): - """Setter for pattern. - - Args: - val: The value to set. - """ - with debug.log_time(log.completion, 'Setting filter pattern'): - self.pattern = val - val = re.sub(r' +', r' ', val) # See #1919 - val = re.escape(val) - val = val.replace(r'\ ', '.*') - self.pattern_re = re.compile(val, re.IGNORECASE) - self.invalidate() - sortcol = 0 - self.sort(sortcol) - - def setSourceModel(self, model): - """Override QSortFilterProxyModel's setSourceModel to clear pattern.""" - log.completion.debug("Setting source model: {}".format(model)) - self.set_pattern('') - super().setSourceModel(model) - self.srcmodel = model - - def filterAcceptsRow(self, row, parent): - """Custom filter implementation. - - Override QSortFilterProxyModel::filterAcceptsRow. - - Args: - row: The row of the item. - parent: The parent item QModelIndex. - - Return: - True if self.pattern is contained in item, or if it's a root item - (category). False in all other cases - """ - if not self.pattern: - return True - - for col in self.columns_to_filter: - idx = self.srcmodel.index(row, col, parent) - if not idx.isValid(): # pragma: no cover - # this is a sanity check not hit by any test case - continue - data = self.srcmodel.data(idx) - if not data: - continue - elif self.pattern_re.search(data): - return True - return False - - def lessThan(self, lindex, rindex): - """Custom sorting implementation. - - Prefers all items which start with self.pattern. Other than that, uses - normal Python string sorting. - - Args: - lindex: The QModelIndex of the left item (*left* < right) - rindex: The QModelIndex of the right item (left < *right*) - - Return: - True if left < right, else False - """ - qtutils.ensure_valid(lindex) - qtutils.ensure_valid(rindex) - - left = self.srcmodel.data(lindex) - right = self.srcmodel.data(rindex) - - leftstart = left.startswith(self.pattern) - rightstart = right.startswith(self.pattern) - - if leftstart and rightstart: - return left < right - elif leftstart: - return True - elif rightstart: - return False - else: - return left < right diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 12e0534c1..4f767abbf 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -29,36 +29,48 @@ from qutebrowser.misc import sql class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" - def __init__(self, name, *, sort_by, sort_order, select, where, - columns_to_filter, parent=None): + def __init__(self, name, *, sort_by=None, sort_order=None, select='*', + where=None, parent=None): + """Create a new completion category backed by a sql table. + + Args: + name: Name of category, and the table in the database. + select: A custom result column expression for the select statement. + where: An optional clause to filter out some rows. + sort_by: The name of the field to sort by, or None for no sorting. + sort_order: Either 'asc' or 'desc', if sort_by is non-None + """ super().__init__(parent=parent) self.name = name + self._sort_by = sort_by + self._sort_order = sort_order + self._select = select + self._where = where + self.set_pattern('', [0]) - query = sql.run_query('select * from {} limit 1'.format(name)) - self._fields = [query.record().fieldName(i) for i in columns_to_filter] + def set_pattern(self, pattern, columns_to_filter): + query = sql.run_query('select * from {} limit 1'.format(self.name)) + fields = [query.record().fieldName(i) for i in columns_to_filter] - querystr = 'select {} from {} where ('.format(select, name) + querystr = 'select {} from {} where ('.format(self._select, self.name) # the incoming pattern will have literal % and _ escaped with '\' # we need to tell sql to treat '\' as an escape character querystr += ' or '.join("{} like ? escape '\\'".format(f) - for f in self._fields) + for f in fields) querystr += ')' - if where: - querystr += ' and ' + where + if self._where: + querystr += ' and ' + self._where - if sort_by: - assert sort_order == 'asc' or sort_order == 'desc' - querystr += ' order by {} {}'.format(sort_by, sort_order) + if self._sort_by: + assert self._sort_order in ['asc', 'desc'] + querystr += ' order by {} {}'.format(self._sort_by, + self._sort_order) - self._querystr = querystr - self.set_pattern('') - - def set_pattern(self, pattern): # escape to treat a user input % or _ as a literal, not a wildcard pattern = pattern.replace('%', '\\%') pattern = pattern.replace('_', '\\_') # treat spaces as wildcards to match any of the typed words pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) - query = sql.run_query(self._querystr, [pattern for _ in self._fields]) + query = sql.run_query(querystr, [pattern for _ in fields]) self.setQuery(query) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 5b738c503..ea3f3f610 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -19,7 +19,8 @@ """Function to return the url completion model for the `open` command.""" -from qutebrowser.completion.models import completionmodel +from qutebrowser.completion.models import (completionmodel, listcategory, + sqlcategory) from qutebrowser.config import config from qutebrowser.utils import qtutils, log, objreg @@ -46,8 +47,7 @@ def _delete_url(completion): log.completion.debug('Deleting bookmark {}'.format(urlstr)) bookmark_manager = objreg.get('bookmark-manager') bookmark_manager.delete(urlstr) - else: - assert catname == 'Quickmarks', 'Unknown category {}'.format(catname) + elif catname == 'Quickmarks': quickmark_manager = objreg.get('quickmark-manager') sibling = index.sibling(index.row(), _TEXTCOL) qtutils.ensure_valid(sibling) @@ -66,15 +66,17 @@ def url(): columns_to_filter=[_URLCOL, _TEXTCOL], delete_cur_item=_delete_url) - quickmarks = objreg.get('quickmark-manager').marks.items() - model.add_list('Quickmarks', ((url, name) for (name, url) in quickmarks)) + quickmarks = ((url, name) for (name, url) + in objreg.get('quickmark-manager').marks.items()) + bookmarks = objreg.get('bookmark-manager').marks.items() - model.add_list('Bookmarks', objreg.get('bookmark-manager').marks.items()) + model.add_category(listcategory.ListCategory('Quickmarks', quickmarks)) + model.add_category(listcategory.ListCategory('Bookmarks', bookmarks)) timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) - model.add_sqltable('History', - sort_order='desc', sort_by='atime', - select='url, title, {}'.format(select_time), - where='not redirect') + hist_cat = sqlcategory.SqlCategory( + 'History', sort_order='desc', sort_by='atime', + select='url, title, {}'.format(select_time), where='not redirect') + model.add_category(hist_cat) return model diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 3b1611a1c..47e304a73 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -158,8 +158,6 @@ PERFECT_FILES = [ 'completion/models/base.py'), ('tests/unit/completion/test_models.py', 'completion/models/urlmodel.py'), - ('tests/unit/completion/test_sortfilter.py', - 'completion/models/sortfilter.py'), ] diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 0b69ad89f..87da45ce6 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -239,6 +239,24 @@ def host_blocker_stub(stubs): objreg.delete('host-blocker') +@pytest.fixture +def quickmark_manager_stub(stubs): + """Fixture which provides a fake quickmark manager object.""" + stub = stubs.QuickmarkManagerStub() + objreg.register('quickmark-manager', stub) + yield stub + objreg.delete('quickmark-manager') + + +@pytest.fixture +def bookmark_manager_stub(stubs): + """Fixture which provides a fake bookmark manager object.""" + stub = stubs.BookmarkManagerStub() + objreg.register('bookmark-manager', stub) + yield stub + objreg.delete('bookmark-manager') + + @pytest.fixture def session_manager_stub(stubs): """Fixture which provides a fake web-history object.""" diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index dce105c27..c407aeab8 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -193,11 +193,10 @@ def test_update_completion(txt, kind, pattern, pos_args, status_command_stub, completer_obj.schedule_completion_update() assert completion_widget_stub.set_model.call_count == 1 args = completion_widget_stub.set_model.call_args[0] - # the outer model is just for sorting; srcmodel is the completion model if kind is None: assert args[0] is None else: - model = args[0].srcmodel + model = args[0] assert model.kind == kind assert model.pos_args == pos_args assert args[1] == pattern diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 4c1241d12..7a61c0b84 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -19,128 +19,59 @@ """Tests for CompletionModel.""" +import sys import pytest +import hypothesis +from unittest import mock +from hypothesis import strategies -from qutebrowser.completion.models import completionmodel, sortfilter +from qutebrowser.completion.models import completionmodel -def _create_model(data, filter_cols=None): - """Create a completion model populated with the given data. - - data: A list of lists, where each sub-list represents a category, each - tuple in the sub-list represents an item, and each value in the - tuple represents the item data for that column - filter_cols: Columns to filter, or None for default. - """ - model = completionmodel.CompletionModel(columns_to_filter=filter_cols) - for catdata in data: - model.add_list('', catdata) - return model +@hypothesis.given(strategies.lists(min_size=0, max_size=3, + elements=strategies.integers(min_value=0, max_value=2**31))) +def test_first_last_item(counts): + """Test that first() and last() index to the first and last items.""" + model = completionmodel.CompletionModel() + for c in counts: + cat = mock.Mock() + cat.rowCount = mock.Mock(return_value=c) + model.add_category(cat) + nonempty = [i for i, rowCount in enumerate(counts) if rowCount > 0] + if not nonempty: + # with no items, first and last should be an invalid index + assert not model.first_item().isValid() + assert not model.last_item().isValid() + else: + first = nonempty[0] + last = nonempty[-1] + # first item of the first nonempty category + assert model.first_item().row() == 0 + assert model.first_item().parent().row() == first + # last item of the last nonempty category + assert model.last_item().row() == counts[last] - 1 + assert model.last_item().parent().row() == last -def _extract_model_data(model): - """Express a model's data as a list for easier comparison. - - Return: A list of lists, where each sub-list represents a category, each - tuple in the sub-list represents an item, and each value in the - tuple represents the item data for that column - """ - data = [] - for i in range(0, model.rowCount()): - cat_idx = model.index(i, 0) - row = [] - for j in range(0, model.rowCount(cat_idx)): - row.append((model.data(cat_idx.child(j, 0)), - model.data(cat_idx.child(j, 1)), - model.data(cat_idx.child(j, 2)))) - data.append(row) - return data +@hypothesis.given(strategies.lists(elements=strategies.integers(), + min_size=0, max_size=3)) +def test_count(counts): + model = completionmodel.CompletionModel() + for c in counts: + cat = mock.Mock(spec=['rowCount']) + cat.rowCount = mock.Mock(return_value=c) + model.add_category(cat) + assert model.count() == sum(counts) -@pytest.mark.parametrize('tree, first, last', [ - ([[('Aa',)]], 'Aa', 'Aa'), - ([[('Aa',)], [('Ba',)]], 'Aa', 'Ba'), - ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], - 'Aa', 'Ca'), - ([[], [('Ba',)]], 'Ba', 'Ba'), - ([[], [], [('Ca',)]], 'Ca', 'Ca'), - ([[], [], [('Ca',), ('Cb',)]], 'Ca', 'Cb'), - ([[('Aa',)], []], 'Aa', 'Aa'), - ([[('Aa',)], []], 'Aa', 'Aa'), - ([[('Aa',)], [], []], 'Aa', 'Aa'), - ([[('Aa',)], [], [('Ca',)]], 'Aa', 'Ca'), - ([[], []], None, None), -]) -def test_first_last_item(tree, first, last): - """Test that first() and last() return indexes to the first and last items. - - Args: - tree: Each list represents a completion category, with each string - being an item under that category. - first: text of the first item - last: text of the last item - """ - model = _create_model(tree) - assert model.data(model.first_item()) == first - assert model.data(model.last_item()) == last - - -@pytest.mark.parametrize('tree, expected', [ - ([[('Aa',)]], 1), - ([[('Aa',)], [('Ba',)]], 2), - ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], 6), - ([[], [('Ba',)]], 1), - ([[], [], [('Ca',)]], 1), - ([[], [], [('Ca',), ('Cb',)]], 2), - ([[('Aa',)], []], 1), - ([[('Aa',)], []], 1), - ([[('Aa',)], [], []], 1), - ([[('Aa',)], [], [('Ca',)]], 2), -]) -def test_count(tree, expected): - model = _create_model(tree) - assert model.count() == expected - - -@pytest.mark.parametrize('pattern, filter_cols, before, after', [ - ('foo', [0], - [[('foo', '', ''), ('bar', '', '')]], - [[('foo', '', '')]]), - - ('foo', [0], - [[('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')]], - [[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]]), - - ('foo', [0], - [[('foo', '', '')], [('bar', '', '')]], - [[('foo', '', '')], []]), - - # prefer foobar as it starts with the pattern - ('foo', [0], - [[('barfoo', '', ''), ('foobar', '', '')]], - [[('foobar', '', ''), ('barfoo', '', '')]]), - - # however, don't rearrange categories - ('foo', [0], - [[('barfoo', '', '')], [('foobar', '', '')]], - [[('barfoo', '', '')], [('foobar', '', '')]]), - - ('foo', [1], - [[('foo', 'bar', ''), ('bar', 'foo', '')]], - [[('bar', 'foo', '')]]), - - ('foo', [0, 1], - [[('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')]], - [[('foo', 'bar', ''), ('bar', 'foo', '')]]), - - ('foo', [0, 1, 2], - [[('foo', '', ''), ('bar', '')]], - [[('foo', '', '')]]), -]) -def test_set_pattern(pattern, filter_cols, before, after): +@hypothesis.given(strategies.text()) +def test_set_pattern(pat): """Validate the filtering and sorting results of set_pattern.""" - # TODO: just test that it calls the mock on its child categories - model = _create_model(before, filter_cols) - model.set_pattern(pattern) - actual = _extract_model_data(model) - assert actual == after + cols = [1, 2, 3] + model = completionmodel.CompletionModel(columns_to_filter=cols) + cats = [mock.Mock(spec=['set_pattern'])] * 3 + for c in cats: + c.set_pattern = mock.Mock() + model.add_category(c) + model.set_pattern(pat) + assert all(c.set_pattern.called_with([pat, cols]) for c in cats) diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 3db4dca5a..43722af42 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -19,13 +19,13 @@ """Tests for the CompletionView Object.""" -import unittest.mock +from unittest import mock import pytest from PyQt5.QtGui import QStandardItem, QColor from qutebrowser.completion import completionwidget -from qutebrowser.completion.models import completionmodel +from qutebrowser.completion.models import completionmodel, listcategory from qutebrowser.commands import cmdexc @@ -70,19 +70,11 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry, return view -@pytest.fixture -def simplemodel(completionview): - """A completion model with one item.""" - model = completionmodel.CompletionModel() - model.add_list('', [('foo'),]) - return model - - def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" model = completionmodel.CompletionModel() for i in range(3): - model.add_list(str(i), [('foo',)]) + cat = listcategory.ListCategory('', [('foo',)]) completionview.set_model(model) assert completionview.model() is model for i in range(model.rowCount()): @@ -91,7 +83,7 @@ def test_set_model(completionview): def test_set_pattern(completionview): model = completionmodel.CompletionModel() - model.set_pattern = unittest.mock.Mock() + model.set_pattern = mock.Mock() completionview.set_model(model, 'foo') model.set_pattern.assert_called_with('foo') @@ -158,7 +150,8 @@ def test_completion_item_focus(which, tree, expected, completionview, qtbot): """ model = completionmodel.CompletionModel() for catdata in tree: - model.add_list('', (x,) for x in catdata) + cat = listcategory.ListCategory('', ((x,) for x in catdata)) + model.add_category(cat) completionview.set_model(model) for entry in expected: if entry is None: @@ -203,7 +196,8 @@ def test_completion_show(show, rows, quick_complete, completionview, model = completionmodel.CompletionModel() for name in rows: - model.add_list('', [(name,)]) + cat = listcategory.ListCategory('', [(name,)]) + model.add_category(cat) assert not completionview.isVisible() completionview.set_model(model) @@ -217,27 +211,33 @@ def test_completion_show(show, rows, quick_complete, completionview, assert not completionview.isVisible() -def test_completion_item_del(completionview, simplemodel): +def test_completion_item_del(completionview): """Test that completion_item_del invokes delete_cur_item in the model.""" - simplemodel.srcmodel.delete_cur_item = unittest.mock.Mock() - completionview.set_model(simplemodel) + func = mock.Mock() + model = completionmodel.CompletionModel(delete_cur_item=func) + model.add_category(listcategory.ListCategory('', [('foo',)])) + completionview.set_model(model) completionview.completion_item_focus('next') completionview.completion_item_del() - assert simplemodel.srcmodel.delete_cur_item.called + assert func.called -def test_completion_item_del_no_selection(completionview, simplemodel): +def test_completion_item_del_no_selection(completionview): """Test that completion_item_del with no selected index.""" - simplemodel.srcmodel.delete_cur_item = unittest.mock.Mock() - completionview.set_model(simplemodel) + func = mock.Mock() + model = completionmodel.CompletionModel(delete_cur_item=func) + model.add_category(listcategory.ListCategory('', [('foo',)])) + completionview.set_model(model) with pytest.raises(cmdexc.CommandError): completionview.completion_item_del() - assert not simplemodel.srcmodel.delete_cur_item.called + assert not func.called -def test_completion_item_del_no_func(completionview, simplemodel): +def test_completion_item_del_no_func(completionview): """Test completion_item_del with no delete_cur_item in the model.""" - completionview.set_model(simplemodel) + model = completionmodel.CompletionModel() + model.add_category(listcategory.ListCategory('', [('foo',)])) + completionview.set_model(model) completionview.completion_item_focus('next') with pytest.raises(cmdexc.CommandError): completionview.completion_item_del() diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py new file mode 100644 index 000000000..0b4426bdb --- /dev/null +++ b/tests/unit/completion/test_listcategory.py @@ -0,0 +1,70 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2016 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Tests for CompletionFilterModel.""" + +import pytest + +from qutebrowser.completion.models import listcategory + + +def _validate(cat, expected): + """Check that a category contains the expected items in the given order. + + Args: + cat: The category to inspect. + expected: A list of tuples containing the expected items. + """ + assert cat.rowCount() == len(expected) + for row, items in enumerate(expected): + for col, item in enumerate(items): + assert cat.data(cat.index(row, col)) == item + + +@pytest.mark.parametrize('pattern, filter_cols, before, after', [ + ('foo', [0], + [('foo', '', ''), ('bar', '', '')], + [('foo', '', '')]), + + ('foo', [0], + [('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')], + [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]), + + # prefer foobar as it starts with the pattern + ('foo', [0], + [('barfoo', '', ''), ('foobar', '', '')], + [('foobar', '', ''), ('barfoo', '', '')]), + + ('foo', [1], + [('foo', 'bar', ''), ('bar', 'foo', '')], + [('bar', 'foo', '')]), + + ('foo', [0, 1], + [('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')], + [('foo', 'bar', ''), ('bar', 'foo', '')]), + + ('foo', [0, 1, 2], + [('foo', '', ''), ('bar', '')], + [('foo', '', '')]), +]) +def test_set_pattern(pattern, filter_cols, before, after): + """Validate the filtering and sorting results of set_pattern.""" + cat = listcategory.ListCategory('Foo', before) + cat.set_pattern(pattern, filter_cols) + _validate(cat, after) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 6e7e60958..48b3646bf 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -134,27 +134,25 @@ def _mock_view_index(model, category_num, child_num, qtbot): @pytest.fixture -def quickmarks(init_sql): - """Pre-populate the quickmark database.""" - table = sql.SqlTable('Quickmarks', ['name', 'url'], primary_key='name') - table.insert(['aw', 'https://wiki.archlinux.org']) - table.insert(['ddg', 'https://duckduckgo.com']) - table.insert(['wiki', 'https://wikipedia.org']) - objreg.register('quickmark-manager', table) - yield table - objreg.delete('quickmark-manager') +def quickmarks(quickmark_manager_stub): + """Pre-populate the quickmark-manager stub with some quickmarks.""" + quickmark_manager_stub.marks = collections.OrderedDict([ + ('aw', 'https://wiki.archlinux.org'), + ('ddg', 'https://duckduckgo.com'), + ('wiki', 'https://wikipedia.org'), + ]) + return quickmark_manager_stub @pytest.fixture -def bookmarks(init_sql): - """Pre-populate the bookmark database.""" - table = sql.SqlTable('Bookmarks', ['url', 'title'], primary_key='url') - table.insert(['https://github.com', 'GitHub']) - table.insert(['https://python.org', 'Welcome to Python.org']) - table.insert(['http://qutebrowser.org', 'qutebrowser | qutebrowser']) - objreg.register('bookmark-manager', table) - yield table - objreg.delete('bookmark-manager') +def bookmarks(bookmark_manager_stub): + """Pre-populate the bookmark-manager stub with some quickmarks.""" + bookmark_manager_stub.marks = collections.OrderedDict([ + ('https://github.com', 'GitHub'), + ('https://python.org', 'Welcome to Python.org'), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser'), + ]) + return bookmark_manager_stub @pytest.fixture @@ -315,9 +313,9 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, # delete item (1, 0) -> (bookmarks, 'https://github.com' ) view = _mock_view_index(model, 1, 0, qtbot) model.delete_cur_item(view) - assert 'https://github.com' not in bookmarks - assert 'https://python.org' in bookmarks - assert 'http://qutebrowser.org' in bookmarks + assert 'https://github.com' not in bookmarks.marks + assert 'https://python.org' in bookmarks.marks + assert 'http://qutebrowser.org' in bookmarks.marks def test_url_completion_delete_quickmark(qtmodeltester, config_stub, @@ -332,9 +330,9 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, # delete item (0, 1) -> (quickmarks, 'ddg' ) view = _mock_view_index(model, 0, 1, qtbot) model.delete_cur_item(view) - assert 'aw' in quickmarks - assert 'ddg' not in quickmarks - assert 'wiki' in quickmarks + assert 'aw' in quickmarks.marks + assert 'ddg' not in quickmarks.marks + assert 'wiki' in quickmarks.marks def test_session_completion(qtmodeltester, session_manager_stub): diff --git a/tests/unit/completion/test_sortfilter.py b/tests/unit/completion/test_sortfilter.py deleted file mode 100644 index 7dadaa9dd..000000000 --- a/tests/unit/completion/test_sortfilter.py +++ /dev/null @@ -1,146 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2015-2017 Florian Bruhin (The Compiler) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Tests for CompletionFilterModel.""" - -import pytest - -from qutebrowser.completion.models import listcategory, sortfilter - - -# TODO: merge listcategory and sortfilter -def _create_model(data): - """Create a completion model populated with the given data. - - data: A list of lists, where each sub-list represents a category, each - tuple in the sub-list represents an item, and each value in the - tuple represents the item data for that column - """ - model = completionmodel.CompletionModel() - for catdata in data: - cat = model.add_list(itemdata) - return model - - -def _extract_model_data(model): - """Express a model's data as a list for easier comparison. - - Return: A list of lists, where each sub-list represents a category, each - tuple in the sub-list represents an item, and each value in the - tuple represents the item data for that column - """ - data = [] - for i in range(0, model.rowCount()): - cat_idx = model.index(i, 0) - row = [] - for j in range(0, model.rowCount(cat_idx)): - row.append((model.data(cat_idx.child(j, 0)), - model.data(cat_idx.child(j, 1)), - model.data(cat_idx.child(j, 2)))) - data.append(row) - return data - - -@pytest.mark.parametrize('pattern, data, expected', [ - ('foo', 'barfoobar', True), - ('foo bar', 'barfoobar', True), - ('foo bar', 'barfoobar', True), - ('foo bar', 'barfoobazbar', True), - ('foo bar', 'barfoobazbar', True), - ('foo', 'barFOObar', True), - ('Foo', 'barfOObar', True), - ('ab', 'aonebtwo', False), - ('33', 'l33t', True), - ('x', 'blah', False), - ('4', 'blah', False), -]) -def test_filter_accepts_row(pattern, data, expected): - source_model = completionmodel.CompletionModel() - cat = source_model.new_category('test') - source_model.new_item(cat, data) - - filter_model = sortfilter.CompletionFilterModel(source_model) - filter_model.set_pattern(pattern) - assert filter_model.rowCount() == 1 # "test" category - idx = filter_model.index(0, 0) - assert idx.isValid() - - row_count = filter_model.rowCount(idx) - assert row_count == (1 if expected else 0) - - -def test_set_source_model(): - """Ensure setSourceModel sets source_model and clears the pattern.""" - model1 = base.CompletionModel() - model2 = base.CompletionModel() - filter_model = sortfilter.CompletionFilterModel(model1) - filter_model.set_pattern('foo') - # sourceModel() is cached as srcmodel, so make sure both match - assert filter_model.srcmodel is model1 - assert filter_model.sourceModel() is model1 - assert filter_model.pattern == 'foo' - filter_model.setSourceModel(model2) - assert filter_model.srcmodel is model2 - assert filter_model.sourceModel() is model2 - assert not filter_model.pattern - - -@pytest.mark.parametrize('pattern, filter_cols, before, after', [ - ('foo', [0], - [[('foo', '', ''), ('bar', '', '')]], - [[('foo', '', '')]]), - - ('foo', [0], - [[('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')]], - [[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]]), - - ('foo', [0], - [[('foo', '', '')], [('bar', '', '')]], - [[('foo', '', '')], []]), - - # prefer foobar as it starts with the pattern - ('foo', [0], - [[('barfoo', '', ''), ('foobar', '', '')]], - [[('foobar', '', ''), ('barfoo', '', '')]]), - - # however, don't rearrange categories - ('foo', [0], - [[('barfoo', '', '')], [('foobar', '', '')]], - [[('barfoo', '', '')], [('foobar', '', '')]]), - - ('foo', [1], - [[('foo', 'bar', ''), ('bar', 'foo', '')]], - [[('bar', 'foo', '')]]), - - ('foo', [0, 1], - [[('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')]], - [[('foo', 'bar', ''), ('bar', 'foo', '')]]), - - ('foo', [0, 1, 2], - [[('foo', '', ''), ('bar', '')]], - [[('foo', '', '')]]), -]) -def test_set_pattern(pattern, filter_cols, before, after): - """Validate the filtering and sorting results of set_pattern.""" - model = _create_model(before) - model.columns_to_filter = filter_cols - filter_model = sortfilter.CompletionFilterModel(model) - filter_model.set_pattern(pattern) - actual = _extract_model_data(filter_model) - assert actual == after diff --git a/tests/unit/completion/test_sqlcateogry.py b/tests/unit/completion/test_sqlcateogry.py new file mode 100644 index 000000000..1d25c5954 --- /dev/null +++ b/tests/unit/completion/test_sqlcateogry.py @@ -0,0 +1,156 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Test SQL-based completions.""" + +import pytest + +from qutebrowser.misc import sql +from qutebrowser.completion.models import sqlcategory + + +pytestmark = pytest.mark.usefixtures('init_sql') + + +def _validate(cat, expected): + """Check that a category contains the expected items in the given order. + + Args: + cat: The category to inspect. + expected: A list of tuples containing the expected items. + """ + assert cat.rowCount() == len(expected) + for row, items in enumerate(expected): + for col, item in enumerate(items): + assert cat.data(cat.index(row, col)) == item + + +@pytest.mark.parametrize('sort_by, sort_order, data, expected', [ + (None, 'asc', + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), + + ('a', 'asc', + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), + + ('a', 'desc', + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), + + ('b', 'asc', + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), + + ('b', 'desc', + [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], + [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), + + ('c', 'asc', + [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], + [('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]), + + ('c', 'desc', + [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], + [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), +]) +def test_sorting(sort_by, sort_order, data, expected): + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + for row in data: + table.insert(row) + cat = sqlcategory.SqlCategory('Foo', sort_by=sort_by, + sort_order=sort_order) + _validate(cat, expected) + + +@pytest.mark.parametrize('pattern, filter_cols, before, after', [ + ('foo', [0], + [('foo', '', ''), ('bar', '', ''), ('aafobbb', '', '')], + [('foo',)]), + + ('foo', [0], + [('baz', 'bar', 'foo'), ('foo', '', ''), ('bar', 'foo', '')], + [('foo', '', '')]), + + ('foo', [0], + [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')], + [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]), + + ('foo', [1], + [('foo', 'bar', ''), ('bar', 'foo', '')], + [('bar', 'foo', '')]), + + ('foo', [0, 1], + [('foo', 'bar', ''), ('bar', 'foo', ''), ('biz', 'baz', 'foo')], + [('foo', 'bar', ''), ('bar', 'foo', '')]), + + ('foo', [0, 1, 2], + [('foo', '', ''), ('bar', '', ''), ('baz', 'bar', 'foo')], + [('foo', '', ''), ('baz', 'bar', 'foo')]), + + ('foo bar', [0], + [('foo', '', ''), ('bar foo', '', ''), ('xfooyybarz', '', '')], + [('xfooyybarz', '', '')]), + + ('foo%bar', [0], + [('foo%bar', '', ''), ('foo bar', '', ''), ('foobar', '', '')], + [('foo%bar', '', '')]), + + ('_', [0], + [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')], + [('a_b', '', ''), ('__a', '', '')]), + + ('%', [0, 1], + [('\\foo', '\\bar', '')], + []), + + ("can't", [0], + [("can't touch this", '', ''), ('a', '', '')], + [("can't touch this", '', '')]), +]) +def test_set_pattern(pattern, filter_cols, before, after): + """Validate the filtering and sorting results of set_pattern.""" + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + for row in before: + table.insert(row) + cat = sqlcategory.SqlCategory('Foo') + cat.set_pattern(pattern, filter_cols) + _validate(cat, after) + + +def test_select(): + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table.insert(['foo', 'bar', 'baz']) + cat = sqlcategory.SqlCategory('Foo', select='b, c, a') + _validate(cat, [('bar', 'baz', 'foo')]) + + +def test_where(): + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table.insert(['foo', 'bar', False]) + table.insert(['baz', 'biz', True]) + cat = sqlcategory.SqlCategory('Foo', where='not c') + _validate(cat, [('foo', 'bar', False)]) + + +def test_entry(): + table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + assert hasattr(table.Entry, 'a') + assert hasattr(table.Entry, 'b') + assert hasattr(table.Entry, 'c') diff --git a/tests/unit/completion/test_sqlmodel.py b/tests/unit/completion/test_sqlmodel.py deleted file mode 100644 index 8b9727dfd..000000000 --- a/tests/unit/completion/test_sqlmodel.py +++ /dev/null @@ -1,230 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Tests for the base sql completion model.""" - -import pytest - -from qutebrowser.misc import sql -from qutebrowser.completion.models import sqlmodel - - -pytestmark = pytest.mark.usefixtures('init_sql') - - -def _check_model(model, expected): - """Check that a model contains the expected items in the given order. - - Args: - expected: A list of form - [ - (cat, [(name, desc, misc), (name, desc, misc), ...]), - (cat, [(name, desc, misc), (name, desc, misc), ...]), - ... - ] - """ - assert model.rowCount() == len(expected) - for i, (expected_title, expected_items) in enumerate(expected): - catidx = model.index(i, 0) - assert model.data(catidx) == expected_title - assert model.rowCount(catidx) == len(expected_items) - for j, (name, desc, misc) in enumerate(expected_items): - assert model.data(model.index(j, 0, catidx)) == name - assert model.data(model.index(j, 1, catidx)) == desc - assert model.data(model.index(j, 2, catidx)) == misc - - -@pytest.mark.parametrize('rowcounts, expected', [ - ([0], 0), - ([1], 1), - ([2], 2), - ([0, 0], 0), - ([0, 0, 0], 0), - ([1, 1], 2), - ([3, 2, 1], 6), - ([0, 2, 0], 2), -]) -def test_count(rowcounts, expected): - model = sqlmodel.SqlCompletionModel() - for i, rowcount in enumerate(rowcounts): - name = 'Foo' + str(i) - table = sql.SqlTable(name, ['a'], primary_key='a') - for rownum in range(rowcount): - table.insert([rownum]) - model.new_category(name) - assert model.count() == expected - - -@pytest.mark.parametrize('sort_by, sort_order, data, expected', [ - (None, 'asc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), - - ('a', 'asc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - - ('a', 'desc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - - ('b', 'asc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - - ('b', 'desc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - - ('c', 'asc', - [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], - [('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]), - - ('c', 'desc', - [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], - [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), -]) -def test_sorting(sort_by, sort_order, data, expected): - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') - for row in data: - table.insert(row) - model = sqlmodel.SqlCompletionModel() - model.new_category('Foo', sort_by=sort_by, sort_order=sort_order) - _check_model(model, [('Foo', expected)]) - - -@pytest.mark.parametrize('pattern, filter_cols, before, after', [ - ('foo', [0], - [('A', [('foo', '', ''), ('bar', '', ''), ('aafobbb', '', '')])], - [('A', [('foo', '', '')])]), - - ('foo', [0], - [('A', [('baz', 'bar', 'foo'), ('foo', '', ''), ('bar', 'foo', '')])], - [('A', [('foo', '', '')])]), - - ('foo', [0], - [('A', [('foo', '', ''), ('bar', '', '')]), - ('B', [('foo', '', ''), ('bar', '', '')])], - [('A', [('foo', '', '')]), ('B', [('foo', '', '')])]), - - ('foo', [0], - [('A', [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')])], - [('A', [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')])]), - - ('foo', [0], - [('A', [('foo', '', '')]), ('B', [('bar', '', '')])], - [('A', [('foo', '', '')]), ('B', [])]), - - ('foo', [1], - [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])], - [('A', [('bar', 'foo', '')])]), - - ('foo', [0, 1], - [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])], - [('A', [('foo', 'bar', ''), ('bar', 'foo', '')])]), - - ('foo', [0, 1, 2], - [('A', [('foo', '', ''), ('bar', '', '')])], - [('A', [('foo', '', '')])]), - - ('foo bar', [0], - [('A', [('foo', '', ''), ('bar foo', '', ''), ('xfooyybarz', '', '')])], - [('A', [('xfooyybarz', '', '')])]), - - ('foo%bar', [0], - [('A', [('foo%bar', '', ''), ('foo bar', '', ''), ('foobar', '', '')])], - [('A', [('foo%bar', '', '')])]), - - ('_', [0], - [('A', [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')])], - [('A', [('a_b', '', ''), ('__a', '', '')])]), - - ('%', [0, 1], - [('A', [('\\foo', '\\bar', '')])], - [('A', [])]), - - ("can't", [0], - [('A', [("can't touch this", '', ''), ('a', '', '')])], - [('A', [("can't touch this", '', '')])]), -]) -def test_set_pattern(pattern, filter_cols, before, after): - """Validate the filtering and sorting results of set_pattern.""" - model = sqlmodel.SqlCompletionModel(columns_to_filter=filter_cols) - for name, rows in before: - table = sql.SqlTable(name, ['a', 'b', 'c'], primary_key='a') - for row in rows: - table.insert(row) - model.new_category(name) - model.set_pattern(pattern) - _check_model(model, after) - - -@pytest.mark.parametrize('data, first, last', [ - ([('A', ['Aa'])], 'Aa', 'Aa'), - ([('A', ['Aa', 'Ba'])], 'Aa', 'Ba'), - ([('A', ['Aa', 'Ab', 'Ac']), ('B', ['Ba', 'Bb']), - ('C', ['Ca'])], 'Aa', 'Ca'), - ([('A', []), ('B', ['Ba'])], 'Ba', 'Ba'), - ([('A', []), ('B', []), ('C', ['Ca'])], 'Ca', 'Ca'), - ([('A', []), ('B', []), ('C', ['Ca', 'Cb'])], 'Ca', 'Cb'), - ([('A', ['Aa']), ('B', [])], 'Aa', 'Aa'), - ([('A', ['Aa']), ('B', []), ('C', [])], 'Aa', 'Aa'), - ([('A', ['Aa']), ('B', []), ('C', ['Ca'])], 'Aa', 'Ca'), - ([('A', []), ('B', [])], None, None), -]) -def test_first_last_item(data, first, last): - """Test that first() and last() return indexes to the first and last items. - - Args: - data: Input to _make_model - first: text of the first item - last: text of the last item - """ - model = sqlmodel.SqlCompletionModel() - for name, rows in data: - table = sql.SqlTable(name, ['a'], primary_key='a') - for row in rows: - table.insert([row]) - model.new_category(name) - assert model.data(model.first_item()) == first - assert model.data(model.last_item()) == last - - -def test_select(): - table = sql.SqlTable('test_select', ['a', 'b', 'c'], primary_key='a') - table.insert(['foo', 'bar', 'baz']) - model = sqlmodel.SqlCompletionModel() - model.new_category('test_select', select='b, c, a') - _check_model(model, [('test_select', [('bar', 'baz', 'foo')])]) - - -def test_where(): - table = sql.SqlTable('test_where', ['a', 'b', 'c'], primary_key='a') - table.insert(['foo', 'bar', False]) - table.insert(['baz', 'biz', True]) - model = sqlmodel.SqlCompletionModel() - model.new_category('test_where', where='not c') - _check_model(model, [('test_where', [('foo', 'bar', False)])]) - - -def test_entry(): - table = sql.SqlTable('test_entry', ['a', 'b', 'c'], primary_key='a') - assert hasattr(table.Entry, 'a') - assert hasattr(table.Entry, 'b') - assert hasattr(table.Entry, 'c') From 5dce6fa49415e083d19ef7fa63a725a54cc19ed3 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 21 Mar 2017 20:42:32 -0400 Subject: [PATCH 052/337] Fix pylint/flake8 errors --- qutebrowser/completion/models/completionmodel.py | 2 -- qutebrowser/completion/models/listcategory.py | 3 ++- tests/unit/completion/test_completionmodel.py | 2 -- tests/unit/completion/test_completionwidget.py | 6 +++--- tests/unit/completion/test_models.py | 1 - 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 84dd74b8f..0ee81fdf5 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -19,8 +19,6 @@ """A model that proxies access to one or more completion categories.""" -import re - from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from qutebrowser.utils import log, qtutils diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index 6251a7cac..d24fd74d0 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -25,7 +25,7 @@ Module attributes: import re -from PyQt5.QtCore import Qt, QSortFilterProxyModel, QModelIndex +from PyQt5.QtCore import QSortFilterProxyModel from PyQt5.QtGui import QStandardItem, QStandardItemModel from qutebrowser.utils import qtutils, debug, log @@ -41,6 +41,7 @@ class ListCategory(QSortFilterProxyModel): self.srcmodel = QStandardItemModel(parent=self) self.pattern = '' self.pattern_re = None + self.columns_to_filter = None for item in items: self.srcmodel.appendRow([QStandardItem(x) for x in item]) self.setSourceModel(self.srcmodel) diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 7a61c0b84..05af902a3 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -19,8 +19,6 @@ """Tests for CompletionModel.""" -import sys -import pytest import hypothesis from unittest import mock from hypothesis import strategies diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 43722af42..7985bfa5b 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -22,7 +22,7 @@ from unittest import mock import pytest -from PyQt5.QtGui import QStandardItem, QColor +from PyQt5.QtGui import QColor from qutebrowser.completion import completionwidget from qutebrowser.completion.models import completionmodel, listcategory @@ -74,10 +74,10 @@ def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" model = completionmodel.CompletionModel() for i in range(3): - cat = listcategory.ListCategory('', [('foo',)]) + model.add_category(listcategory.ListCategory('', [('foo',)])) completionview.set_model(model) assert completionview.model() is model - for i in range(model.rowCount()): + for i in range(3): assert completionview.isExpanded(model.index(i, 0)) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 48b3646bf..fa228847e 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -30,7 +30,6 @@ from qutebrowser.browser import history from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value from qutebrowser.misc import sql -from qutebrowser.utils import objreg def _check_completions(model, expected): From 93e0bfa410a429c680a8eba563d8ce3fa1d2b08e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 21 Mar 2017 21:41:13 -0400 Subject: [PATCH 053/337] Fix tests after sql completion rebase. --- qutebrowser/browser/qutescheme.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 23b009f6d..d12fb8f8e 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -212,7 +212,7 @@ def history_data(start_time): # noqa for item in history: # Skip redirects # Skip qute:// links - if item.redirect or item.url.scheme() == 'qute': + if item.redirect or item.url.startswith('qute://'): continue # Skip items out of time window @@ -241,11 +241,10 @@ def history_data(start_time): # noqa return # Use item's url as title if there's no title. - item_url = item.url.toDisplayString() - item_title = item.title if item.title else item_url + item_title = item.title if item.title else item.url item_time = int(item.atime * 1000) - yield {"url": item_url, "title": item_title, "time": item_time} + yield {"url": item.url, "title": item_title, "time": item_time} # if we reached here, we had reached the end of history yield {"next": int(last_item.atime if last_item else -1)} From 4296eed429289b463177e11ead28af9c294474bd Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 22 Mar 2017 19:55:03 -0400 Subject: [PATCH 054/337] Fix test_history cleanup failure. The test may be skipped if the PyQt5.QtWebKitWidget import fails, but the cleanup was still running and trying to delete a nonexistant web-history. --- tests/unit/browser/webkit/test_history.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 9d8711152..84af5fb46 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -325,12 +325,12 @@ def test_history_interface(qtbot, webview, hist_interface): @pytest.fixture def cleanup_init(): - yield # prevent test_init from leaking state - hist = objreg.get('web-history') - hist.setParent(None) - objreg.delete('web-history') + yield try: + hist = objreg.get('web-history') + hist.setParent(None) + objreg.delete('web-history') from PyQt5.QtWebKit import QWebHistoryInterface QWebHistoryInterface.setDefaultInterface(None) except: From 7d04f155c8a15b588a265af63d96445cd949c992 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 23 Mar 2017 08:29:22 -0400 Subject: [PATCH 055/337] Add missing docstring. --- qutebrowser/completion/models/sqlcategory.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 4f767abbf..693aaa841 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -49,6 +49,12 @@ class SqlCategory(QSqlQueryModel): self.set_pattern('', [0]) def set_pattern(self, pattern, columns_to_filter): + """Set the pattern used to filter results. + + Args: + pattern: string pattern to filter by. + columns_to_filter: indices of columns to apply pattern to. + """ query = sql.run_query('select * from {} limit 1'.format(self.name)) fields = [query.record().fieldName(i) for i in columns_to_filter] From 2eea115b3acd2c22d943d4c20850c244ff63bda0 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 23 Mar 2017 08:31:09 -0400 Subject: [PATCH 056/337] Rename sqlcategory and add to perfect_files. There was a typo in the file name. --- scripts/dev/check_coverage.py | 2 ++ .../completion/{test_sqlcateogry.py => test_sqlcategory.py} | 0 2 files changed, 2 insertions(+) rename tests/unit/completion/{test_sqlcateogry.py => test_sqlcategory.py} (100%) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 47e304a73..9904edd5b 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -158,6 +158,8 @@ PERFECT_FILES = [ 'completion/models/base.py'), ('tests/unit/completion/test_models.py', 'completion/models/urlmodel.py'), + ('tests/unit/completion/test_sqlcategory.py', + 'completion/models/sqlcategory.py'), ] diff --git a/tests/unit/completion/test_sqlcateogry.py b/tests/unit/completion/test_sqlcategory.py similarity index 100% rename from tests/unit/completion/test_sqlcateogry.py rename to tests/unit/completion/test_sqlcategory.py From b47c3b6a601b5ba1e8d9854c607867ca00523e78 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 23 Mar 2017 12:02:07 -0400 Subject: [PATCH 057/337] Test deleting a history entry from completion. Deleting a history entry should do nothing, but we want a test to ensure this and get 100% branch coverage for urlmodel. This also un-skips the bookmark/quickmark tests. --- tests/unit/completion/test_models.py | 30 ++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index fa228847e..9bc6fb289 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -167,6 +167,7 @@ def web_history(stubs, init_sql): datetime(2016, 3, 8).timestamp(), False]) table.insert(['https://github.com', 'https://github.com', datetime(2016, 5, 1).timestamp(), False]) + return table def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, @@ -235,7 +236,6 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): }) -@pytest.mark.skip def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() @@ -244,14 +244,13 @@ def test_quickmark_completion(qtmodeltester, quickmarks): _check_completions(model, { "Quickmarks": [ - ('aw', 'https://wiki.archlinux.org', ''), - ('ddg', 'https://duckduckgo.com', ''), - ('wiki', 'https://wikipedia.org', ''), + ('aw', 'https://wiki.archlinux.org', None), + ('ddg', 'https://duckduckgo.com', None), + ('wiki', 'https://wikipedia.org', None), ] }) -@pytest.mark.skip def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() @@ -260,9 +259,9 @@ def test_bookmark_completion(qtmodeltester, bookmarks): _check_completions(model, { "Bookmarks": [ - ('https://github.com', 'GitHub', ''), - ('https://python.org', 'Welcome to Python.org', ''), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', ''), + ('https://github.com', 'GitHub', None), + ('https://python.org', 'Welcome to Python.org', None), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ] }) @@ -334,6 +333,21 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, assert 'wiki' in quickmarks.marks +def test_url_completion_delete_history(qtmodeltester, config_stub, + web_history, quickmarks, bookmarks, + qtbot): + """Test that deleting a history entry is a noop.""" + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} + model = urlmodel.url() + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + hist_before = list(web_history) + view = _mock_view_index(model, 2, 0, qtbot) + model.delete_cur_item(view) + assert list(web_history) == hist_before + + def test_session_completion(qtmodeltester, session_manager_stub): session_manager_stub.sessions = ['default', '1', '2'] model = miscmodels.session() From de5be0dc5a1873704bd9971663bca2912eaf6f4e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 26 Mar 2017 16:52:10 -0400 Subject: [PATCH 058/337] Store history in an on-disk sqlite database. Instead of reading sqlite history from a file and storing it in an in-memory database, just directly use an on-disk database. This resolves #755, where history entries don't pop in to the completion menu immediately as they are still being read asynchronously for a few seconds after the browser starts. --- qutebrowser/app.py | 21 +-- qutebrowser/browser/history.py | 81 +--------- qutebrowser/misc/sql.py | 21 ++- scripts/dev/check_coverage.py | 2 + tests/helpers/fixtures.py | 5 +- tests/unit/browser/webkit/test_history.py | 176 +++++++--------------- tests/unit/misc/test_sql.py | 17 ++- 7 files changed, 92 insertions(+), 231 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 33726b058..0f98a9216 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -158,8 +158,6 @@ def init(args, crash_handler): QDesktopServices.setUrlHandler('https', open_desktopservices_url) QDesktopServices.setUrlHandler('qute', open_desktopservices_url) - QTimer.singleShot(10, functools.partial(_init_late_modules, args)) - log.init.debug("Init done!") crash_handler.raise_crashdlg() @@ -428,7 +426,7 @@ def _init_modules(args, crash_handler): keyconf.init(qApp) log.init.debug("Initializing sql...") - sql.init() + sql.init(os.path.join(standarddir.data(), 'history.sqlite')) log.init.debug("Initializing web history...") history.init(qApp) @@ -476,23 +474,6 @@ def _init_modules(args, crash_handler): browsertab.init() -def _init_late_modules(args): - """Initialize modules which can be inited after the window is shown.""" - log.init.debug("Reading web history...") - reader = objreg.get('web-history').async_read() - with debug.log_time(log.init, 'Reading history'): - while True: - QApplication.processEvents() - try: - next(reader) - except StopIteration: - break - except (OSError, UnicodeDecodeError) as e: - error.handle_fatal_exc(e, args, "Error while initializing!", - pre_text="Error while initializing") - sys.exit(usertypes.Exit.err_init) - - class Quitter: """Utility class to quit/restart the QApplication. diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index b6d328942..e7ff54037 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -26,7 +26,7 @@ from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl from qutebrowser.commands import cmdutils from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils, usertypes, message) -from qutebrowser.misc import lineparser, objects, sql +from qutebrowser.misc import objects, sql class Entry: @@ -118,14 +118,6 @@ class WebHistory(sql.SqlTable): self._saved_count tracks how many of those entries were already written to disk, so we can always append to the existing data. - Attributes: - _lineparser: The AppendLineParser used to save the history. - _new_history: A list of Entry items of the current session. - _saved_count: How many HistoryEntries have been written to disk. - _initial_read_started: Whether async_read was called. - _initial_read_done: Whether async_read has completed. - _temp_history: List of history entries from before async_read finished. - Signals: cleared: Emitted after the history is cleared. """ @@ -133,59 +125,13 @@ class WebHistory(sql.SqlTable): cleared = pyqtSignal() async_read_done = pyqtSignal() - def __init__(self, hist_dir, hist_name, parent=None): + def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], primary_key='url', parent=parent) - self._initial_read_started = False - self._initial_read_done = False - self._lineparser = lineparser.AppendLineParser(hist_dir, hist_name, - parent=self) - self._temp_history = [] - self._new_history = [] - self._saved_count = 0 def __repr__(self): return utils.get_repr(self, length=len(self)) - def async_read(self): - """Read the initial history.""" - if self._initial_read_started: - log.init.debug("Ignoring async_read() because reading is started.") - return - self._initial_read_started = True - - with self._lineparser.open(): - for line in self._lineparser: - yield - - line = line.rstrip() - if not line: - continue - - try: - entry = Entry.from_str(line) - except ValueError as e: - log.init.warning("Invalid history entry {!r}: {}!".format( - line, e)) - continue - - # This de-duplicates history entries; only the latest - # entry for each URL is kept. If you want to keep - # information about previous hits change the items in - # old_urls to be lists or change Entry to have a - # list of atimes. - self._add_entry(entry) - - self._initial_read_done = True - self.async_read_done.emit() - objreg.get('save-manager').add_saveable( - 'history', self.save, self.changed) - - for entry in self._temp_history: - self._add_entry(entry) - self._new_history.append(entry) - self._temp_history.clear() - def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, entry.atime, @@ -193,15 +139,7 @@ class WebHistory(sql.SqlTable): def get_recent(self): """Get the most recent history entries.""" - old = self._lineparser.get_recent() - return old + [str(e) for e in self._new_history] - - def save(self): - """Save the history to disk.""" - new = (str(e) for e in self._new_history[self._saved_count:]) - self._lineparser.new_data = new - self._lineparser.save() - self._saved_count = len(self._new_history) + return self.select(sort_by='atime', sort_order='desc', limit=100) @cmdutils.register(name='history-clear', instance='web-history') def clear(self, force=False): @@ -221,11 +159,7 @@ class WebHistory(sql.SqlTable): "history?") def _do_clear(self): - self._lineparser.clear() self.delete_all() - self._temp_history.clear() - self._new_history.clear() - self._saved_count = 0 self.cleared.emit() @pyqtSlot(QUrl, QUrl, str) @@ -263,11 +197,7 @@ class WebHistory(sql.SqlTable): if atime is None: atime = time.time() entry = Entry(atime, url, title, redirect=redirect) - if self._initial_read_done: - self._add_entry(entry) - self._new_history.append(entry) - else: - self._temp_history.append(entry) + self._add_entry(entry) def init(parent=None): @@ -276,8 +206,7 @@ def init(parent=None): Args: parent: The parent to use for WebHistory. """ - history = WebHistory(hist_dir=standarddir.data(), hist_name='history', - parent=parent) + history = WebHistory(parent=parent) objreg.register('web-history', history) if objects.backend == usertypes.Backend.QtWebKit: diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index c9249d362..eb1ad533c 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -34,14 +34,13 @@ class SqlException(Exception): pass -def init(): +def init(db_path): """Initialize the SQL database connection.""" database = QSqlDatabase.addDatabase('QSQLITE') - # In-memory database, see https://sqlite.org/inmemorydb.html - database.setDatabaseName(':memory:') + database.setDatabaseName(db_path) if not database.open(): - raise SqlException("Failed to open in-memory sqlite database: {}" - .format(database.lastError().text())) + raise SqlException("Failed to open sqlite database at {}: {}" + .format(db_path, database.lastError().text())) def close(): @@ -103,8 +102,8 @@ class SqlTable(QObject): super().__init__(parent) self._name = name self._primary_key = primary_key - run_query("CREATE TABLE {} ({}, PRIMARY KEY ({}))".format( - name, ','.join(fields), primary_key)) + run_query("CREATE TABLE IF NOT EXISTS {} ({}, PRIMARY KEY ({}))" + .format(name, ','.join(fields), primary_key)) # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) @@ -177,3 +176,11 @@ class SqlTable(QObject): """Remove all row from the table.""" run_query("DELETE FROM {}".format(self._name)) self.changed.emit() + + def select(self, sort_by, sort_order, limit): + """Remove all row from the table.""" + result = run_query('SELECT * FROM {} ORDER BY {} {} LIMIT {}' + .format(self._name, sort_by, sort_order, limit)) + while result.next(): + rec = result.record() + yield self.Entry(*[rec.value(i) for i in range(rec.count())]) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 9904edd5b..a2950b658 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -55,6 +55,8 @@ PERFECT_FILES = [ 'browser/history.py'), ('tests/unit/browser/webkit/test_history.py', 'browser/webkit/webkithistory.py'), + ('tests/unit/browser/webkit/test_history.py', + 'browser/history.py'), ('tests/unit/browser/webkit/http/test_http.py', 'browser/webkit/http.py'), ('tests/unit/browser/webkit/http/test_content_disposition.py', diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 87da45ce6..24087ed18 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -476,8 +476,9 @@ def short_tmpdir(): @pytest.fixture -def init_sql(): +def init_sql(data_tmpdir): """Initialize the SQL module, and shut it down after the test.""" - sql.init() + path = str(data_tmpdir / 'test.db') + sql.init(path) yield sql.close() diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 84af5fb46..9335cb239 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -38,47 +38,17 @@ def prerequisites(config_stub, fake_save_manager, init_sql): @pytest.fixture() def hist(tmpdir): - return history.WebHistory(hist_dir=str(tmpdir), hist_name='history') + return history.WebHistory() -def test_register_saveable(monkeypatch, qtbot, tmpdir, caplog, - fake_save_manager): - (tmpdir / 'filled-history').write('12345 http://example.com/ title') - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) - assert fake_save_manager.add_saveable.called - - -def test_async_read_twice(monkeypatch, qtbot, tmpdir, caplog): - (tmpdir / 'filled-history').write('\n'.join([ - '12345 http://example.com/ title', - '67890 http://example.com/', - '12345 http://qutebrowser.org/ blah', - ])) - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - next(hist.async_read()) - with pytest.raises(StopIteration): - next(hist.async_read()) - expected = "Ignoring async_read() because reading is started." - assert expected in (record.msg for record in caplog.records) - - -@pytest.mark.parametrize('redirect', [True, False]) -def test_adding_item_during_async_read(qtbot, hist, redirect): - """Check what happens when adding URL while reading the history.""" - url = 'http://www.example.com/' - hist.add_url(QUrl(url), redirect=redirect, atime=12345) - - with qtbot.waitSignal(hist.async_read_done): - list(hist.async_read()) - - assert not hist._temp_history - assert list(hist) == [(url, '', 12345, redirect)] +@pytest.fixture() +def mock_time(mocker): + m = mocker.patch('qutebrowser.browser.history.time') + m.time.return_value = 12345 + return 12345 def test_iter(hist): - list(hist.async_read()) - urlstr = 'http://www.example.com/' url = QUrl(urlstr) hist.add_url(url, atime=12345) @@ -88,7 +58,6 @@ def test_iter(hist): def test_len(hist): assert len(hist) == 0 - list(hist.async_read()) url = QUrl('http://www.example.com/') hist.add_url(url) @@ -96,101 +65,49 @@ def test_len(hist): assert len(hist) == 1 -@pytest.mark.parametrize('line', [ - '12345 http://example.com/ title', # with title - '67890 http://example.com/', # no title - '12345 http://qutebrowser.org/ ', # trailing space - ' ', - '', -]) -def test_read(tmpdir, line): - (tmpdir / 'filled-history').write(line + '\n') - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) +def test_updated_entries(tmpdir, hist): + hist.add_url(QUrl('http://example.com/'), atime=67890) + assert list(hist) == [('http://example.com/', '', 67890, False)] - -def test_updated_entries(tmpdir): - (tmpdir / 'filled-history').write('12345 http://example.com/\n' - '67890 http://example.com/\n') - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) - - assert hist['http://example.com/'] == ('http://example.com/', '', 67890, - False) hist.add_url(QUrl('http://example.com/'), atime=99999) - assert hist['http://example.com/'] == ('http://example.com/', '', 99999, - False) + assert list(hist) == [('http://example.com/', '', 99999, False)] -def test_invalid_read(tmpdir, caplog): - (tmpdir / 'filled-history').write('foobar\n12345 http://example.com/') - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - with caplog.at_level(logging.WARNING): - list(hist.async_read()) - - entries = list(hist) - - assert len(entries) == 1 - msg = "Invalid history entry 'foobar': 2 or 3 fields expected!" - assert msg in (rec.msg for rec in caplog.records) - - -def test_get_recent(tmpdir): - (tmpdir / 'filled-history').write('12345 http://example.com/') - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) - +def test_get_recent(hist): hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) - lines = hist.get_recent() - - expected = ['12345 http://example.com/', - '67890 http://www.qutebrowser.org/'] - assert lines == expected + hist.add_url(QUrl('http://example.com/'), atime=12345) + assert list(hist.get_recent()) == [ + ('http://www.qutebrowser.org/', '', 67890 , False), + ('http://example.com/', '', 12345, False), + ] -def test_save(tmpdir): - hist_file = tmpdir / 'filled-history' - hist_file.write('12345 http://example.com/\n') - - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) - +def test_save(tmpdir, hist): + hist.add_url(QUrl('http://example.com/'), atime=12345) hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) - hist.save() - lines = hist_file.read().splitlines() - expected = ['12345 http://example.com/', - '67890 http://www.qutebrowser.org/'] - assert lines == expected - - hist.add_url(QUrl('http://www.the-compiler.org/'), atime=99999) - hist.save() - expected.append('99999 http://www.the-compiler.org/') - - lines = hist_file.read().splitlines() - assert lines == expected + hist2 = history.WebHistory() + assert list(hist2) == [('http://example.com/', '', 12345, False), + ('http://www.qutebrowser.org/', '', 67890, False)] -def test_clear(qtbot, tmpdir): - hist_file = tmpdir / 'filled-history' - hist_file.write('12345 http://example.com/\n') +def test_clear(qtbot, tmpdir, hist, mocker): + hist.add_url(QUrl('http://example.com/')) + hist.add_url(QUrl('http://www.qutebrowser.org/')) - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) + m = mocker.patch('qutebrowser.browser.history.message.confirm_async') + hist.clear() + m.assert_called() + +def test_clear_force(qtbot, tmpdir, hist): + hist.add_url(QUrl('http://example.com/')) hist.add_url(QUrl('http://www.qutebrowser.org/')) with qtbot.waitSignal(hist.cleared): - hist._do_clear() + hist.clear(force=True) - assert not hist_file.read() - assert not hist._new_history - - hist.add_url(QUrl('http://www.the-compiler.org/'), atime=67890) - hist.save() - - lines = hist_file.read().splitlines() - assert lines == ['67890 http://www.the-compiler.org/'] + assert not len(hist) @pytest.mark.parametrize('item', [ @@ -198,21 +115,34 @@ def test_clear(qtbot, tmpdir): ('http://www.example.com', 12346, 'the title', True) ]) def test_add_item(qtbot, hist, item): - list(hist.async_read()) (url, atime, title, redirect) = item hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) assert hist[url] == (url, title, atime, redirect) -def test_add_item_redirect_update(qtbot, tmpdir, fake_save_manager): +def test_add_item_invalid(qtbot, hist, caplog): + with caplog.at_level(logging.WARNING): + hist.add_url(QUrl()) + assert not list(hist) + + +@pytest.mark.parametrize('level, url, req_url, expected', [ + (logging.DEBUG, 'a.com', 'a.com', [('a.com', 'title', 12345, False)]), + (logging.DEBUG, 'a.com', 'b.com', [('a.com', 'title', 12345, False), + ('b.com', 'title', 12345, True)]), + (logging.WARNING, 'a.com', '', [('a.com', 'title', 12345, False)]), + (logging.WARNING, '', '', []), +]) +def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog): + with caplog.at_level(level): + hist.add_from_tab(QUrl(url), QUrl(req_url), 'title') + assert set(list(hist)) == set(expected) + + +def test_add_item_redirect_update(qtbot, tmpdir, hist): """A redirect update added should override a non-redirect one.""" url = 'http://www.example.com/' - - hist_file = tmpdir / 'filled-history' - hist_file.write('12345 {}\n'.format(url)) - hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history') - list(hist.async_read()) - + hist.add_url(QUrl(url), atime=5555) hist.add_url(QUrl(url), redirect=True, atime=67890) assert hist[url] == (url, '', 67890, True) diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index f4898b25d..4221bb022 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -28,9 +28,8 @@ pytestmark = pytest.mark.usefixtures('init_sql') def test_init(): sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - with pytest.raises(sql.SqlException): - # table name collision on 'Foo' - sql.SqlTable('Foo', ['foo', 'bar'], primary_key='foo') + # should not error if table already exists + sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') def test_insert(qtbot): @@ -54,6 +53,18 @@ def test_iter(): ('thirteen', 13, True)] +@pytest.mark.parametrize('rows, sort_by, sort_order, limit, result', [ + ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', 5, [(1, 6), (2, 5), (3, 4)]), + ([[2, 5], [1, 6], [3, 4]], 'a', 'desc', 3, [(3, 4), (2, 5), (1, 6)]), + ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]) +]) +def test_select(rows, sort_by, sort_order, limit, result): + table = sql.SqlTable('Foo', ['a', 'b'], primary_key='a') + for row in rows: + table.insert(row) + assert list(table.select(sort_by, sort_order, limit)) == result + + def test_replace(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') table.insert(['one', 1, False]) From 80647b062a6f28002dd283c8460a4ca7baed92fd Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 29 Mar 2017 08:59:16 -0400 Subject: [PATCH 059/337] Convert old history file to sqlite. If qutebrowser detects a history text file when it starts (~/.local/share/qutebrowser/history by default on Linux), it will import this file into the new sqlite database, then delete it. The read is done as a coroutine as it can take some time. --- qutebrowser/app.py | 23 +++++ qutebrowser/browser/history.py | 74 ++++++++------- qutebrowser/misc/sql.py | 52 +++++++++-- tests/unit/browser/webkit/test_history.py | 105 +++++----------------- 4 files changed, 135 insertions(+), 119 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 0f98a9216..e2e2d5107 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -158,6 +158,8 @@ def init(args, crash_handler): QDesktopServices.setUrlHandler('https', open_desktopservices_url) QDesktopServices.setUrlHandler('qute', open_desktopservices_url) + _import_history() + log.init.debug("Init done!") crash_handler.raise_crashdlg() @@ -474,6 +476,27 @@ def _init_modules(args, crash_handler): browsertab.init() +def _import_history(): + """Import a history text file into sqlite if it exists. + + In older versions of qutebrowser, history was stored in a text format. + This converts that file into the new sqlite format and removes it. + """ + path = os.path.join(standarddir.data(), 'history') + if not os.path.isfile(path): + return + + def action(): + with debug.log_time(log.init, 'Converting old history file to sqlite'): + objreg.get('web-history').read(path) + message.info('History import complete. Removing {}'.format(path)) + os.remove(path) + + # delay to give message time to appear before locking down for import + message.info('Converting {} to sqlite...'.format(path)) + QTimer.singleShot(100, action) + + class Quitter: """Utility class to quit/restart the QApplication. diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e7ff54037..8d10cdbbe 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -20,6 +20,7 @@ """Simple history which gets written to disk.""" import time +import os from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl @@ -70,37 +71,6 @@ class Entry: """Get the URL as a lossless string.""" return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) - @classmethod - def from_str(cls, line): - """Parse a history line like '12345 http://example.com title'.""" - data = line.split(maxsplit=2) - if len(data) == 2: - atime, url = data - title = "" - elif len(data) == 3: - atime, url, title = data - else: - raise ValueError("2 or 3 fields expected") - - url = QUrl(url) - if not url.isValid(): - raise ValueError("Invalid URL: {}".format(url.errorString())) - - # https://github.com/qutebrowser/qutebrowser/issues/670 - atime = atime.lstrip('\0') - - if '-' in atime: - atime, flags = atime.split('-') - else: - flags = '' - - if not set(flags).issubset('r'): - raise ValueError("Invalid flags {!r}".format(flags)) - - redirect = 'r' in flags - - return cls(atime, url, title, redirect=redirect) - class WebHistory(sql.SqlTable): @@ -199,6 +169,48 @@ class WebHistory(sql.SqlTable): entry = Entry(atime, url, title, redirect=redirect) self._add_entry(entry) + def _parse_entry(self, line): + """Parse a history line like '12345 http://example.com title'.""" + data = line.split(maxsplit=2) + if len(data) == 2: + atime, url = data + title = "" + elif len(data) == 3: + atime, url, title = data + else: + raise ValueError("2 or 3 fields expected") + + url = QUrl(url) + if not url.isValid(): + raise ValueError("Invalid URL: {}".format(url.errorString())) + + # https://github.com/qutebrowser/qutebrowser/issues/670 + atime = atime.lstrip('\0') + + if '-' in atime: + atime, flags = atime.split('-') + else: + flags = '' + + if not set(flags).issubset('r'): + raise ValueError("Invalid flags {!r}".format(flags)) + + redirect = 'r' in flags + + return (url, title, float(atime), bool(redirect)) + + def read(self, path): + """Import a text file into the sql database.""" + with open(path, 'r') as f: + rows = [] + for line in f: + try: + row = self._parse_entry(line.strip()) + rows.append(row) + except ValueError: + log.init.warning('Skipping history line {}'.format(line)) + self.insert_batch(rows) + def init(parent=None): """Initialize the web history. diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index eb1ad533c..d1d478302 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -55,16 +55,21 @@ def version(): return result.record().value(0) +def _prepare_query(querystr): + log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) + database = QSqlDatabase.database() + query = QSqlQuery(database) + query.prepare(querystr) + return query + + def run_query(querystr, values=None): """Run the given SQL query string on the database. Args: values: A list of positional parameter bindings. """ - log.sql.debug('Running SQL query: "{}"'.format(querystr)) - database = QSqlDatabase.database() - query = QSqlQuery(database) - query.prepare(querystr) + query = _prepare_query(querystr) for val in values or []: query.addBindValue(val) log.sql.debug('Query bindings: {}'.format(query.boundValues())) @@ -74,6 +79,28 @@ def run_query(querystr, values=None): return query +def run_batch(querystr, values): + """Run the given SQL query string on the database in batch mode. + + Args: + values: A list of lists, where each inner list contains positional + bindings for one run of the batch. + """ + query = _prepare_query(querystr) + transposed = [list(row) for row in zip(*values)] + for val in transposed: + query.addBindValue(val) + log.sql.debug('Batch Query bindings: {}'.format(query.boundValues())) + + db = QSqlDatabase.database() + db.transaction() + if not query.execBatch(): + raise SqlException('Failed to exec query "{}": "{}"'.format( + querystr, query.lastError().text())) + db.commit() + + return query + class SqlTable(QObject): """Interface to a sql table. @@ -102,8 +129,8 @@ class SqlTable(QObject): super().__init__(parent) self._name = name self._primary_key = primary_key - run_query("CREATE TABLE IF NOT EXISTS {} ({}, PRIMARY KEY ({}))" - .format(name, ','.join(fields), primary_key)) + run_query("CREATE TABLE IF NOT EXISTS {} ({})" + .format(name, ','.join(fields))) # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) @@ -172,6 +199,19 @@ class SqlTable(QObject): values) self.changed.emit() + def insert_batch(self, rows, replace=False): + """Performantly append multiple rows to the table. + + Args: + rows: A list of lists, where each sub-list is a row. + replace: If true, allow inserting over an existing primary key. + """ + cmd = "REPLACE" if replace else "INSERT" + paramstr = ','.join(['?'] * len(rows[0])) + run_batch("{} INTO {} values({})".format(cmd, self._name, paramstr), + rows) + self.changed.emit() + def delete_all(self): """Remove all row from the table.""" run_query("DELETE FROM {}".format(self._name)) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 9335cb239..e86950a82 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -148,88 +148,6 @@ def test_add_item_redirect_update(qtbot, tmpdir, hist): assert hist[url] == (url, '', 67890, True) -@pytest.mark.parametrize('line, expected', [ - ( - # old format without title - '12345 http://example.com/', - history.Entry(atime=12345, url=QUrl('http://example.com/'), title='',) - ), - ( - # trailing space without title - '12345 http://example.com/ ', - history.Entry(atime=12345, url=QUrl('http://example.com/'), title='',) - ), - ( - # new format with title - '12345 http://example.com/ this is a title', - history.Entry(atime=12345, url=QUrl('http://example.com/'), - title='this is a title') - ), - ( - # weird NUL bytes - '\x0012345 http://example.com/', - history.Entry(atime=12345, url=QUrl('http://example.com/'), title=''), - ), - ( - # redirect flag - '12345-r http://example.com/ this is a title', - history.Entry(atime=12345, url=QUrl('http://example.com/'), - title='this is a title', redirect=True) - ), -]) -def test_entry_parse_valid(line, expected): - entry = history.Entry.from_str(line) - assert entry == expected - - -@pytest.mark.parametrize('line', [ - '12345', # one field - '12345 ::', # invalid URL - 'xyz http://www.example.com/', # invalid timestamp - '12345-x http://www.example.com/', # invalid flags - '12345-r-r http://www.example.com/', # double flags -]) -def test_entry_parse_invalid(line): - with pytest.raises(ValueError): - history.Entry.from_str(line) - - -@hypothesis.given(strategies.text()) -def test_entry_parse_hypothesis(text): - """Make sure parsing works or gives us ValueError.""" - try: - history.Entry.from_str(text) - except ValueError: - pass - - -@pytest.mark.parametrize('entry, expected', [ - # simple - ( - history.Entry(12345, QUrl('http://example.com/'), "the title"), - "12345 http://example.com/ the title", - ), - # timestamp as float - ( - history.Entry(12345.678, QUrl('http://example.com/'), "the title"), - "12345 http://example.com/ the title", - ), - # no title - ( - history.Entry(12345.678, QUrl('http://example.com/'), ""), - "12345 http://example.com/", - ), - # redirect flag - ( - history.Entry(12345.678, QUrl('http://example.com/'), "", - redirect=True), - "12345-r http://example.com/", - ), -]) -def test_entry_str(entry, expected): - assert str(entry) == expected - - @pytest.fixture def hist_interface(): # pylint: disable=invalid-name @@ -298,3 +216,26 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): # For this to work, nothing can ever have called setDefaultInterface # before (so we need to test webengine before webkit) assert default_interface is None + + +def test_read(hist, tmpdir, caplog): + histfile = tmpdir / 'history' + histfile.write('''12345 http://example.com/ title + 12346 http://qutebrowser.org/ + 67890 http://example.com/path + + xyz http://example.com/bad-timestamp + 12345 + http://example.com/no-timestamp + 68891-r http://example.com/path/other + 68891-r-r http://example.com/double-flag''') + + with caplog.at_level(logging.WARNING): + hist.read(str(histfile)) + + assert list(hist) == [ + ('http://example.com/', 'title', 12345, False), + ('http://qutebrowser.org/', '', 12346, False), + ('http://example.com/path', '', 67890, False), + ('http://example.com/path/other', '', 68891, True) + ] From 8ff45331df32e35c214662056f8b5d72e789b8eb Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 4 Apr 2017 08:27:42 -0400 Subject: [PATCH 060/337] Clean up sql implementation. Now that sql is only used for history (not quickmarks/bookmarks) a number of functions are no longer needed. In addition, primary key support was removed as we actually need to support multiple entries for the same url with different access times. The completion model will have to handle this by selecting something like (url, title, max(atime)). This also fixes up a number of tests that were broken with the last few sql-related commits. --- qutebrowser/app.py | 2 +- qutebrowser/browser/history.py | 4 +- qutebrowser/misc/sql.py | 45 ++++-------------- qutebrowser/utils/version.py | 3 +- tests/end2end/features/test_history_bdd.py | 38 +++++++-------- tests/unit/browser/test_qutescheme.py | 3 +- tests/unit/browser/webkit/test_history.py | 19 +------- tests/unit/completion/test_models.py | 3 +- tests/unit/completion/test_sqlcategory.py | 10 ++-- tests/unit/misc/test_sql.py | 55 ++++------------------ 10 files changed, 49 insertions(+), 133 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index e2e2d5107..ccb474e69 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -481,7 +481,7 @@ def _import_history(): In older versions of qutebrowser, history was stored in a text format. This converts that file into the new sqlite format and removes it. - """ + """ path = os.path.join(standarddir.data(), 'history') if not os.path.isfile(path): return diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 8d10cdbbe..29e23750b 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -97,7 +97,7 @@ class WebHistory(sql.SqlTable): def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], - primary_key='url', parent=parent) + parent=parent) def __repr__(self): return utils.get_repr(self, length=len(self)) @@ -105,7 +105,7 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, entry.atime, - entry.redirect], replace=True) + entry.redirect]) def get_recent(self): """Get the most recent history entries.""" diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index d1d478302..707e0c4ce 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -101,6 +101,7 @@ def run_batch(querystr, values): return query + class SqlTable(QObject): """Interface to a sql table. @@ -108,7 +109,6 @@ class SqlTable(QObject): Attributes: Entry: The class wrapping row data from this table. _name: Name of the SQL table this wraps. - _primary_key: The primary key of the table. Signals: changed: Emitted when the table is modified. @@ -116,7 +116,7 @@ class SqlTable(QObject): changed = pyqtSignal() - def __init__(self, name, fields, primary_key, parent=None): + def __init__(self, name, fields, parent=None): """Create a new table in the sql database. Raises SqlException if the table already exists. @@ -124,11 +124,9 @@ class SqlTable(QObject): Args: name: Name of the table. fields: A list of field names. - primary_key: Name of the field to serve as the primary key. """ super().__init__(parent) self._name = name - self._primary_key = primary_key run_query("CREATE TABLE IF NOT EXISTS {} ({})" .format(name, ','.join(fields))) # pylint: disable=invalid-name @@ -141,74 +139,47 @@ class SqlTable(QObject): rec = result.record() yield self.Entry(*[rec.value(i) for i in range(rec.count())]) - def __contains__(self, key): - """Return whether the table contains the matching item. - - Args: - key: Primary key value to search for. - """ - query = run_query("SELECT * FROM {} where {} = ?".format( - self._name, self._primary_key), [key]) - return query.next() - def __len__(self): """Return the count of rows in the table.""" result = run_query("SELECT count(*) FROM {}".format(self._name)) result.next() return result.value(0) - def __getitem__(self, key): - """Retrieve the row matching the given key. - - Args: - key: Primary key value to fetch. - """ - result = run_query("SELECT * FROM {} where {} = ?".format( - self._name, self._primary_key), [key]) - result.next() - rec = result.record() - return self.Entry(*[rec.value(i) for i in range(rec.count())]) - - def delete(self, value, field=None): + def delete(self, value, field): """Remove all rows for which `field` equals `value`. Args: value: Key value to delete. - field: Field to use as the key, defaults to the primary key. + field: Field to use as the key. Return: The number of rows deleted. """ - field = field or self._primary_key query = run_query("DELETE FROM {} where {} = ?".format( self._name, field), [value]) if not query.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, values, replace=False): + def insert(self, values): """Append a row to the table. Args: values: A list of values to insert. - replace: If true, allow inserting over an existing primary key. """ - cmd = "REPLACE" if replace else "INSERT" paramstr = ','.join(['?'] * len(values)) - run_query("{} INTO {} values({})".format(cmd, self._name, paramstr), + run_query("INSERT INTO {} values({})".format(self._name, paramstr), values) self.changed.emit() - def insert_batch(self, rows, replace=False): + def insert_batch(self, rows): """Performantly append multiple rows to the table. Args: rows: A list of lists, where each sub-list is a row. - replace: If true, allow inserting over an existing primary key. """ - cmd = "REPLACE" if replace else "INSERT" paramstr = ','.join(['?'] * len(rows[0])) - run_batch("{} INTO {} values({})".format(cmd, self._name, paramstr), + run_batch("INSERT INTO {} values({})".format(self._name, paramstr), rows) self.changed.emit() diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index ac74f997a..faa890152 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -327,7 +327,8 @@ def version(): lines += ['pdf.js: {}'.format(_pdfjs_version())] - sql.init() + # we can use an in-memory database as we just want to query the version + sql.init('') lines += ['sqlite: {}'.format(sql.version())] sql.close() diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 1fee533eb..25995e9d0 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -20,31 +20,29 @@ import os.path import pytest_bdd as bdd + +from PyQt5.QtSql import QSqlDatabase, QSqlQuery + bdd.scenarios('history.feature') @bdd.then(bdd.parsers.parse("the history file should contain:\n{expected}")) def check_history(quteproc, httpbin, expected): - history_file = os.path.join(quteproc.basedir, 'data', 'history') - quteproc.send_cmd(':save history') - quteproc.wait_for(message=':save saved history') - - expected = expected.replace('(port)', str(httpbin.port)).splitlines() - - with open(history_file, 'r', encoding='utf-8') as f: - lines = [] - for line in f: - if not line.strip(): - continue - print('history line: ' + line) - atime, line = line.split(' ', maxsplit=1) - line = line.rstrip() - if '-' in atime: - flags = atime.split('-')[1] - line = '{} {}'.format(flags, line) - lines.append(line) - - assert lines == expected + path = os.path.join(quteproc.basedir, 'data', 'history.sqlite') + db = QSqlDatabase.addDatabase('QSQLITE') + db.setDatabaseName(path) + assert db.open(), 'Failed to open history database' + query = db.exec_('select * from History') + actual = [] + while query.next(): + rec = query.record() + url = rec.value(0) + title = rec.value(1) + redirect = rec.value(3) + actual.append('{} {} {}'.format('r' * redirect, url, title).strip()) + db = None + QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) + assert actual == expected.replace('(port)', str(httpbin.port)).splitlines() @bdd.then("the history file should be empty") diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index d5e60efab..86a322322 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -98,7 +98,7 @@ class TestHistoryHandler: @pytest.fixture def fake_web_history(self, fake_save_manager, tmpdir, init_sql): """Create a fake web-history and register it into objreg.""" - web_history = history.WebHistory(tmpdir.dirname, 'fake-history') + web_history = history.WebHistory() objreg.register('web-history', web_history) yield web_history objreg.delete('web-history') @@ -108,7 +108,6 @@ class TestHistoryHandler: """Create fake history.""" for item in entries: fake_web_history._add_entry(item) - fake_web_history.save() @pytest.mark.parametrize("start_time_offset, expected_item_count", [ (0, 4), diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index e86950a82..50c632f7d 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -65,14 +65,6 @@ def test_len(hist): assert len(hist) == 1 -def test_updated_entries(tmpdir, hist): - hist.add_url(QUrl('http://example.com/'), atime=67890) - assert list(hist) == [('http://example.com/', '', 67890, False)] - - hist.add_url(QUrl('http://example.com/'), atime=99999) - assert list(hist) == [('http://example.com/', '', 99999, False)] - - def test_get_recent(hist): hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) hist.add_url(QUrl('http://example.com/'), atime=12345) @@ -117,7 +109,7 @@ def test_clear_force(qtbot, tmpdir, hist): def test_add_item(qtbot, hist, item): (url, atime, title, redirect) = item hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) - assert hist[url] == (url, title, atime, redirect) + assert list(hist) == [(url, title, atime, redirect)] def test_add_item_invalid(qtbot, hist, caplog): @@ -139,15 +131,6 @@ def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog): assert set(list(hist)) == set(expected) -def test_add_item_redirect_update(qtbot, tmpdir, hist): - """A redirect update added should override a non-redirect one.""" - url = 'http://www.example.com/' - hist.add_url(QUrl(url), atime=5555) - hist.add_url(QUrl(url), redirect=True, atime=67890) - - assert hist[url] == (url, '', 67890, True) - - @pytest.fixture def hist_interface(): # pylint: disable=invalid-name diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 9bc6fb289..e071f1584 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -157,8 +157,7 @@ def bookmarks(bookmark_manager_stub): @pytest.fixture def web_history(stubs, init_sql): """Pre-populate the web-history database.""" - table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect'], - primary_key='url') + table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect']) table.insert(['http://some-redirect.example.com', 'redirect', datetime(2016, 9, 5).timestamp(), True]) table.insert(['http://qutebrowser.org', 'qutebrowser', diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 1d25c5954..03288f016 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -71,7 +71,7 @@ def _validate(cat, expected): [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), ]) def test_sorting(sort_by, sort_order, data, expected): - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: table.insert(row) cat = sqlcategory.SqlCategory('Foo', sort_by=sort_by, @@ -126,7 +126,7 @@ def test_sorting(sort_by, sort_order, data, expected): ]) def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: table.insert(row) cat = sqlcategory.SqlCategory('Foo') @@ -135,14 +135,14 @@ def test_set_pattern(pattern, filter_cols, before, after): def test_select(): - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert(['foo', 'bar', 'baz']) cat = sqlcategory.SqlCategory('Foo', select='b, c, a') _validate(cat, [('bar', 'baz', 'foo')]) def test_where(): - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert(['foo', 'bar', False]) table.insert(['baz', 'biz', True]) cat = sqlcategory.SqlCategory('Foo', where='not c') @@ -150,7 +150,7 @@ def test_where(): def test_entry(): - table = sql.SqlTable('Foo', ['a', 'b', 'c'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b', 'c']) assert hasattr(table.Entry, 'a') assert hasattr(table.Entry, 'b') assert hasattr(table.Entry, 'c') diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 4221bb022..387b5ce46 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -27,24 +27,21 @@ pytestmark = pytest.mark.usefixtures('init_sql') def test_init(): - sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + sql.SqlTable('Foo', ['name', 'val', 'lucky']) # should not error if table already exists - sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + sql.SqlTable('Foo', ['name', 'val', 'lucky']) def test_insert(qtbot): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) with qtbot.waitSignal(table.changed): table.insert(['one', 1, False]) with qtbot.waitSignal(table.changed): table.insert(['wan', 1, False]) - with pytest.raises(sql.SqlException): - # duplicate primary key - table.insert(['one', 1, False]) def test_iter(): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) table.insert(['one', 1, False]) table.insert(['nine', 9, False]) table.insert(['thirteen', 13, True]) @@ -59,29 +56,21 @@ def test_iter(): ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]) ]) def test_select(rows, sort_by, sort_order, limit, result): - table = sql.SqlTable('Foo', ['a', 'b'], primary_key='a') + table = sql.SqlTable('Foo', ['a', 'b']) for row in rows: table.insert(row) assert list(table.select(sort_by, sort_order, limit)) == result -def test_replace(qtbot): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert(['one', 1, False]) - with qtbot.waitSignal(table.changed): - table.insert(['one', 1, True], replace=True) - assert list(table) == [('one', 1, True)] - - def test_delete(qtbot): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) table.insert(['one', 1, False]) table.insert(['nine', 9, False]) table.insert(['thirteen', 13, True]) with pytest.raises(KeyError): - table.delete('nope') + table.delete('nope', 'name') with qtbot.waitSignal(table.changed): - table.delete('thirteen') + table.delete('thirteen', 'name') assert list(table) == [('one', 1, False), ('nine', 9, False)] with qtbot.waitSignal(table.changed): table.delete(False, field='lucky') @@ -89,7 +78,7 @@ def test_delete(qtbot): def test_len(): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) assert len(table) == 0 table.insert(['one', 1, False]) assert len(table) == 1 @@ -99,32 +88,8 @@ def test_len(): assert len(table) == 3 -def test_contains(): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) - assert 'oone' not in table - assert 'ninee' not in table - assert 1 not in table - assert '*' not in table - assert 'one' in table - assert 'nine' in table - assert 'thirteen' in table - - -def test_getitem(): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) - assert table['one'] == ('one', 1, False) - assert table['nine'] == ('nine', 9, False) - assert table['thirteen'] == ('thirteen', 13, True) - - def test_delete_all(qtbot): - table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], primary_key='name') + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) table.insert(['one', 1, False]) table.insert(['nine', 9, False]) table.insert(['thirteen', 13, True]) From 3e63b62d6e43717b1b0c900125bf0a7ba55c9a97 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 6 Apr 2017 12:02:43 -0400 Subject: [PATCH 061/337] Fix pylint/flake8 for sql work. --- qutebrowser/app.py | 4 ++-- qutebrowser/browser/history.py | 6 ++---- tests/end2end/features/test_history_bdd.py | 2 +- tests/unit/browser/webkit/test_history.py | 4 +--- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index ccb474e69..dd73827af 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -479,8 +479,8 @@ def _init_modules(args, crash_handler): def _import_history(): """Import a history text file into sqlite if it exists. - In older versions of qutebrowser, history was stored in a text format. - This converts that file into the new sqlite format and removes it. + In older versions of qutebrowser, history was stored in a text format. + This converts that file into the new sqlite format and removes it. """ path = os.path.join(standarddir.data(), 'history') if not os.path.isfile(path): diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 29e23750b..1a5614fd6 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -20,13 +20,11 @@ """Simple history which gets written to disk.""" import time -import os from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl from qutebrowser.commands import cmdutils -from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils, - usertypes, message) +from qutebrowser.utils import utils, objreg, log, qtutils, usertypes, message from qutebrowser.misc import objects, sql @@ -201,7 +199,7 @@ class WebHistory(sql.SqlTable): def read(self, path): """Import a text file into the sql database.""" - with open(path, 'r') as f: + with open(path, 'r', encoding='utf-8') as f: rows = [] for line in f: try: diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 25995e9d0..70f23f86b 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -21,7 +21,7 @@ import os.path import pytest_bdd as bdd -from PyQt5.QtSql import QSqlDatabase, QSqlQuery +from PyQt5.QtSql import QSqlDatabase bdd.scenarios('history.feature') diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 50c632f7d..1877929b7 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -22,8 +22,6 @@ import logging import pytest -import hypothesis -from hypothesis import strategies from PyQt5.QtCore import QUrl from qutebrowser.browser import history @@ -69,7 +67,7 @@ def test_get_recent(hist): hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) hist.add_url(QUrl('http://example.com/'), atime=12345) assert list(hist.get_recent()) == [ - ('http://www.qutebrowser.org/', '', 67890 , False), + ('http://www.qutebrowser.org/', '', 67890, False), ('http://example.com/', '', 12345, False), ] From 6412c88277db61393f99c19315db5df4bd0593b1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 8 Apr 2017 07:13:16 -0400 Subject: [PATCH 062/337] Clean up history module. Eliminate out-of-date docstring and remove an unused signal. --- qutebrowser/browser/history.py | 20 +------------------- tests/unit/browser/webkit/test_history.py | 5 +---- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 1a5614fd6..6cc98270c 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -72,25 +72,8 @@ class Entry: class WebHistory(sql.SqlTable): - """The global history of visited pages. + """The global history of visited pages.""" - This is a little more complex as you'd expect so the history can be read - from disk async while new history is already arriving. - - While reading from disk is still ongoing, the history is saved in - self._temp_history instead, and then inserted into the sql table once - the async read completes. - - All history which is new in this session (rather than read from disk from a - previous browsing session) is also stored in self._new_history. - self._saved_count tracks how many of those entries were already written to - disk, so we can always append to the existing data. - - Signals: - cleared: Emitted after the history is cleared. - """ - - cleared = pyqtSignal() async_read_done = pyqtSignal() def __init__(self, parent=None): @@ -128,7 +111,6 @@ class WebHistory(sql.SqlTable): def _do_clear(self): self.delete_all() - self.cleared.emit() @pyqtSlot(QUrl, QUrl, str) def add_from_tab(self, url, requested_url, title): diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 1877929b7..22db1490a 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -93,10 +93,7 @@ def test_clear(qtbot, tmpdir, hist, mocker): def test_clear_force(qtbot, tmpdir, hist): hist.add_url(QUrl('http://example.com/')) hist.add_url(QUrl('http://www.qutebrowser.org/')) - - with qtbot.waitSignal(hist.cleared): - hist.clear(force=True) - + hist.clear(force=True) assert not len(hist) From 024386d1892bacf3e3de5526077b4a072cea4a4b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 8 Apr 2017 16:20:55 -0400 Subject: [PATCH 063/337] Fail on history file parsing errors. Instead of skipping bad history lines during the import to sql, fail hard. We don't want to delete the user's old history file if we couldn't parse all of the lines. --- qutebrowser/browser/history.py | 8 +++++-- tests/unit/browser/webkit/test_history.py | 26 ++++++++++++++++------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 6cc98270c..52d32c114 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -183,12 +183,16 @@ class WebHistory(sql.SqlTable): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: rows = [] - for line in f: + for (i, line) in enumerate(f): + line = line.strip() + if not line: + continue try: row = self._parse_entry(line.strip()) rows.append(row) except ValueError: - log.init.warning('Skipping history line {}'.format(line)) + raise Exception('Failed to parse line #{} of {}: "{}"' + .format(i, path, line)) self.insert_batch(rows) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 22db1490a..b2642f7c8 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -196,20 +196,16 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): assert default_interface is None -def test_read(hist, tmpdir, caplog): +def test_read(hist, tmpdir): histfile = tmpdir / 'history' + # empty line is deliberate, to test skipping empty lines histfile.write('''12345 http://example.com/ title 12346 http://qutebrowser.org/ 67890 http://example.com/path - xyz http://example.com/bad-timestamp - 12345 - http://example.com/no-timestamp - 68891-r http://example.com/path/other - 68891-r-r http://example.com/double-flag''') + 68891-r http://example.com/path/other ''') - with caplog.at_level(logging.WARNING): - hist.read(str(histfile)) + hist.read(str(histfile)) assert list(hist) == [ ('http://example.com/', 'title', 12345, False), @@ -217,3 +213,17 @@ def test_read(hist, tmpdir, caplog): ('http://example.com/path', '', 67890, False), ('http://example.com/path/other', '', 68891, True) ] + + +@pytest.mark.parametrize('line', [ + 'xyz http://example.com/bad-timestamp', + '12345', + 'http://example.com/no-timestamp', + '68891-r-r http://example.com/double-flag', +]) +def test_read_invalid(hist, tmpdir, line): + histfile = tmpdir / 'history' + histfile.write(line) + + with pytest.raises(Exception): + hist.read(str(histfile)) From f110cf4d53fd22ba191bfe54a97dfbcc385d2fa1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 8 Apr 2017 17:18:20 -0400 Subject: [PATCH 064/337] Fix long hang after importing history. Turns out historyContains was getting called for the webkit backend multiple times when the browser starts. This was calling `url in history`, which was enumerating the entire history as `__contains__` was not defined. --- qutebrowser/browser/history.py | 3 +++ qutebrowser/misc/sql.py | 11 +++++++++++ tests/unit/browser/webkit/test_history.py | 8 ++++++++ tests/unit/misc/test_sql.py | 16 ++++++++++++++++ 4 files changed, 38 insertions(+) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 52d32c114..aef551e5c 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -83,6 +83,9 @@ class WebHistory(sql.SqlTable): def __repr__(self): return utils.get_repr(self, length=len(self)) + def __contains__(self, url): + return self.contains('url', url) + def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, entry.atime, diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 707e0c4ce..9cbf6024b 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -139,6 +139,17 @@ class SqlTable(QObject): rec = result.record() yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + def contains(self, field, value): + """Return whether the table contains the matching item. + + Args: + field: Field to match. + value: Value to check for the given field. + """ + query = run_query("SELECT * FROM {} where {} = ? LIMIT 1" + .format(self._name, field), [value]) + return query.next() + def __len__(self): """Return the count of rows in the table.""" result = run_query("SELECT count(*) FROM {}".format(self._name)) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index b2642f7c8..ec36f7a88 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -63,6 +63,14 @@ def test_len(hist): assert len(hist) == 1 +def test_contains(hist): + hist.add_url(QUrl('http://www.example.com/'), title='Title', atime=12345) + assert 'http://www.example.com/' in hist + assert not 'www.example.com' in hist + assert not 'Title' in hist + assert not 12345 in hist + + def test_get_recent(hist): hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) hist.add_url(QUrl('http://example.com/'), atime=12345) diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 387b5ce46..edc156a55 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -88,6 +88,22 @@ def test_len(): assert len(table) == 3 +def test_contains(): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) + table.insert(['one', 1, False]) + table.insert(['nine', 9, False]) + table.insert(['thirteen', 13, True]) + assert table.contains('name', 'one') + assert table.contains('name', 'thirteen') + assert table.contains('val', 9) + assert table.contains('lucky', False) + assert table.contains('lucky', True) + assert not table.contains('name', 'oone') + assert not table.contains('name', 1) + assert not table.contains('name', '*') + assert not table.contains('val', 10) + + def test_delete_all(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) table.insert(['one', 1, False]) From e661fb74468236e5d6af32b1f271ffe941367dbb Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 8 Apr 2017 17:24:11 -0400 Subject: [PATCH 065/337] Fix test_history. History doesn't depend on standarddir anymore, the history file path get passed by app.py. --- tests/unit/browser/webkit/test_history.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index ec36f7a88..d444c166c 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -179,7 +179,6 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): else: assert backend == usertypes.Backend.QtWebEngine - monkeypatch.setattr(history.standarddir, 'data', lambda: str(tmpdir)) monkeypatch.setattr(history.objects, 'backend', backend) history.init(qapp) hist = objreg.get('web-history') From 44080b8ad433c73a35058606de8e9fbdf06874b2 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 9 Apr 2017 07:02:14 -0400 Subject: [PATCH 066/337] Fix flake8 errors in test_history --- tests/unit/browser/webkit/test_history.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index d444c166c..4f1a3362d 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -66,9 +66,9 @@ def test_len(hist): def test_contains(hist): hist.add_url(QUrl('http://www.example.com/'), title='Title', atime=12345) assert 'http://www.example.com/' in hist - assert not 'www.example.com' in hist - assert not 'Title' in hist - assert not 12345 in hist + assert 'www.example.com' not in hist + assert 'Title' not in hist + assert 12345 not in hist def test_get_recent(hist): From 231bbe7c2bec0ff432252baa0ba2c39d650d9834 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 11 Apr 2017 07:52:45 -0400 Subject: [PATCH 067/337] Fix race condition on history tests. Two history end2end tests are failing because sqlite is not flushing to disk in time to be read by the test process. My understanding is that sqlite should take an exclusive lock while writing, so it is difficult to understand why this is happening. This can be fixed by adding a delay, but that seems flaky. I'm fixing it by checking qute://history instead of reading the database file. See: https://github.com/qutebrowser/qutebrowser/pull/2295#issuecomment-292786138 and the following discussion. --- tests/end2end/features/history.feature | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 2d02c518a..3ef602712 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -14,7 +14,7 @@ Feature: Page history Then the history file should contain: http://localhost:(port)/data/numbers/1.txt http://localhost:(port)/data/numbers/2.txt - + Scenario: History item with title When I open data/title.html Then the history file should contain: @@ -26,7 +26,7 @@ Feature: Page history Then the history file should contain: r http://localhost:(port)/redirect-to?url=data/title.html Test title http://localhost:(port)/data/title.html Test title - + Scenario: History item with spaces in URL When I open data/title with spaces.html Then the history file should contain: @@ -36,20 +36,24 @@ Feature: Page history When I open data/äöü.html Then the history file should contain: http://localhost:(port)/data/%C3%A4%C3%B6%C3%BC.html Chäschüechli - + + # The following two tests use qute://history instead of checking the + # history file due to a race condition with sqlite. + # https://github.com/qutebrowser/qutebrowser/pull/2295#issuecomment-292786138 @flaky @qtwebengine_todo: Error page message is not implemented Scenario: History with an error When I run :open file:///does/not/exist And I wait for "Error while loading file:///does/not/exist: Error opening /does/not/exist: *" in the log - Then the history file should contain: - file:///does/not/exist Error loading page: file:///does/not/exist + And I open qute://history + Then the page should contain the plaintext "Error loading page: file:///does/not/exist" @qtwebengine_todo: Error page message is not implemented Scenario: History with a 404 When I open status/404 without waiting And I wait for "Error while loading http://localhost:*/status/404: NOT FOUND" in the log - Then the history file should contain: - http://localhost:(port)/status/404 Error loading page: http://localhost:(port)/status/404 + And I open qute://history + Then the page should contain the plaintext "Error loading page: http://localhost:" + And the page should contain the plaintext "/status/404" Scenario: History with invalid URL When I run :tab-only From a8ed9f1c2f8e85b129166ce58ee07d64698d4547 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 11 Apr 2017 08:12:23 -0400 Subject: [PATCH 068/337] Fix qute://version sql init bug. Calling sql.init() in version.version() would replace the existing sql connection and cause a crash when accessed by opening qute://version. Now version relies on sql already being initted, and app.py inits sql early if the --version arg is given. --- qutebrowser/app.py | 3 +++ qutebrowser/utils/version.py | 4 ---- tests/end2end/features/misc.feature | 8 ++++++++ tests/unit/utils/test_version.py | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index dd73827af..0f95c1c5d 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -83,6 +83,9 @@ def run(args): standarddir.init(args) if args.version: + # we need to init sql to print the sql version + # we can use an in-memory database as we just want to query the version + sql.init('') print(version.version()) sys.exit(usertypes.Exit.ok) diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index faa890152..a408ce1fd 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -326,11 +326,7 @@ def version(): lines += _module_versions() lines += ['pdf.js: {}'.format(_pdfjs_version())] - - # we can use an in-memory database as we just want to query the version - sql.init('') lines += ['sqlite: {}'.format(sql.version())] - sql.close() lines += [ 'SSL: {}'.format(QSslSocket.sslLibraryVersionString()), diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index 279df7808..f6c5ce84a 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -702,3 +702,11 @@ Feature: Various utility commands. And I wait for "Renderer process was killed" in the log And I open data/numbers/3.txt Then no crash should happen + And the following tabs should be open: + - data/numbers/3.txt (active) + + ## Other + + Scenario: Open qute://version + When I open qute://version + Then the page should contain the plaintext "Version info" diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 8c0f16081..c0c731f88 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -809,7 +809,7 @@ def test_chromium_version_unpatched(qapp): (True, False, True, True, False), # unknown Linux distribution ]) def test_version_output(git_commit, frozen, style, with_webkit, - known_distribution, stubs, monkeypatch): + known_distribution, stubs, monkeypatch, init_sql): """Test version.version().""" class FakeWebEngineProfile: def httpUserAgent(self): From 784d9bb043213649988ce7a6f2d0568427b2c428 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 11 Apr 2017 09:01:33 -0400 Subject: [PATCH 069/337] Remove code rendered dead by sql implementation. Vulture exposed the following dead code: - AppendLineParse was only used for reading the history text file, which is now a sql database (and the import code for the old text file is simpler and does not need a complex line parser) - async_read_done is no longer used as importing the history text file is synchronous (and should only happen once) - config._init_key_config is unused as it was moved to keyconf.init --- qutebrowser/browser/history.py | 4 +- qutebrowser/config/config.py | 32 ------------ qutebrowser/misc/lineparser.py | 72 +-------------------------- tests/unit/misc/test_lineparser.py | 80 ------------------------------ 4 files changed, 3 insertions(+), 185 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index aef551e5c..d7753cd03 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -21,7 +21,7 @@ import time -from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl +from PyQt5.QtCore import pyqtSlot, QUrl from qutebrowser.commands import cmdutils from qutebrowser.utils import utils, objreg, log, qtutils, usertypes, message @@ -74,8 +74,6 @@ class WebHistory(sql.SqlTable): """The global history of visited pages.""" - async_read_done = pyqtSignal() - def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index e7e74cc19..1245429c2 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -174,38 +174,6 @@ def _init_main_config(parent=None): return -def _init_key_config(parent): - """Initialize the key config. - - Args: - parent: The parent to use for the KeyConfigParser. - """ - from qutebrowser.config.parsers import keyconf - args = objreg.get('args') - try: - key_config = keyconf.KeyConfigParser(standarddir.config(), 'keys.conf', - args.relaxed_config, - parent=parent) - except (keyconf.KeyConfigError, cmdexc.CommandError, - UnicodeDecodeError) as e: - log.init.exception(e) - errstr = "Error while reading key config:\n" - if e.lineno is not None: - errstr += "In line {}: ".format(e.lineno) - error.handle_fatal_exc(e, args, "Error while reading key config!", - pre_text=errstr) - # We didn't really initialize much so far, so we just quit hard. - sys.exit(usertypes.Exit.err_key_config) - else: - objreg.register('key-config', key_config) - save_manager = objreg.get('save-manager') - filename = os.path.join(standarddir.config(), 'keys.conf') - save_manager.add_saveable( - 'key-config', key_config.save, key_config.config_dirty, - config_opt=('general', 'auto-save-config'), filename=filename, - dirty=key_config.is_dirty) - - def _init_misc(): """Initialize misc. config-related files.""" save_manager = objreg.get('save-manager') diff --git a/qutebrowser/misc/lineparser.py b/qutebrowser/misc/lineparser.py index ea9d100b7..2256d9697 100644 --- a/qutebrowser/misc/lineparser.py +++ b/qutebrowser/misc/lineparser.py @@ -21,7 +21,6 @@ import os import os.path -import itertools import contextlib from PyQt5.QtCore import pyqtSlot, pyqtSignal, QObject @@ -96,7 +95,7 @@ class BaseLineParser(QObject): """ assert self._configfile is not None if self._opened: - raise IOError("Refusing to double-open AppendLineParser.") + raise IOError("Refusing to double-open LineParser.") self._opened = True try: if self._binary: @@ -133,73 +132,6 @@ class BaseLineParser(QObject): raise NotImplementedError -class AppendLineParser(BaseLineParser): - - """LineParser which reads lazily and appends data to existing one. - - Attributes: - _new_data: The data which was added in this session. - """ - - def __init__(self, configdir, fname, *, parent=None): - super().__init__(configdir, fname, binary=False, parent=parent) - self.new_data = [] - self._fileobj = None - - def __iter__(self): - if self._fileobj is None: - raise ValueError("Iterating without open() being called!") - file_iter = (line.rstrip('\n') for line in self._fileobj) - return itertools.chain(file_iter, iter(self.new_data)) - - @contextlib.contextmanager - def open(self): - """Open the on-disk history file. Needed for __iter__.""" - try: - with self._open('r') as f: - self._fileobj = f - yield - except FileNotFoundError: - self._fileobj = [] - yield - finally: - self._fileobj = None - - def get_recent(self, count=4096): - """Get the last count bytes from the underlying file.""" - with self._open('r') as f: - f.seek(0, os.SEEK_END) - size = f.tell() - try: - if size - count > 0: - offset = size - count - else: - offset = 0 - f.seek(offset) - data = f.readlines() - finally: - f.seek(0, os.SEEK_END) - return data - - def save(self): - do_save = self._prepare_save() - if not do_save: - return - with self._open('a') as f: - self._write(f, self.new_data) - self.new_data = [] - self._after_save() - - def clear(self): - do_save = self._prepare_save() - if not do_save: - return - with self._open('w'): - pass - self.new_data = [] - self._after_save() - - class LineParser(BaseLineParser): """Parser for configuration files which are simply line-based. @@ -240,7 +172,7 @@ class LineParser(BaseLineParser): def save(self): """Save the config file.""" if self._opened: - raise IOError("Refusing to double-open AppendLineParser.") + raise IOError("Refusing to double-open LineParser.") do_save = self._prepare_save() if not do_save: return diff --git a/tests/unit/misc/test_lineparser.py b/tests/unit/misc/test_lineparser.py index af439c006..adae2485d 100644 --- a/tests/unit/misc/test_lineparser.py +++ b/tests/unit/misc/test_lineparser.py @@ -125,83 +125,3 @@ class TestLineParser: lineparser._prepare_save = lambda: False lineparser.save() assert (tmpdir / 'file').read() == 'pristine\n' - - -class TestAppendLineParser: - - BASE_DATA = ['old data 1', 'old data 2'] - - @pytest.fixture - def lineparser(self, tmpdir): - """Fixture to get an AppendLineParser for tests.""" - lp = lineparsermod.AppendLineParser(str(tmpdir), 'file') - lp.new_data = self.BASE_DATA - lp.save() - return lp - - def _get_expected(self, new_data): - """Get the expected data with newlines.""" - return '\n'.join(self.BASE_DATA + new_data) + '\n' - - def test_save(self, tmpdir, lineparser): - """Test save().""" - new_data = ['new data 1', 'new data 2'] - lineparser.new_data = new_data - lineparser.save() - assert (tmpdir / 'file').read() == self._get_expected(new_data) - - def test_clear(self, tmpdir, lineparser): - """Check if calling clear() empties both pending and persisted data.""" - lineparser.new_data = ['one', 'two'] - lineparser.save() - assert (tmpdir / 'file').read() == "old data 1\nold data 2\none\ntwo\n" - - lineparser.new_data = ['one', 'two'] - lineparser.clear() - lineparser.save() - assert not lineparser.new_data - assert (tmpdir / 'file').read() == "" - - def test_iter_without_open(self, lineparser): - """Test __iter__ without having called open().""" - with pytest.raises(ValueError): - iter(lineparser) - - def test_iter(self, lineparser): - """Test __iter__.""" - new_data = ['new data 1', 'new data 2'] - lineparser.new_data = new_data - with lineparser.open(): - assert list(lineparser) == self.BASE_DATA + new_data - - def test_iter_not_found(self, mocker): - """Test __iter__ with no file.""" - open_mock = mocker.patch( - 'qutebrowser.misc.lineparser.AppendLineParser._open') - open_mock.side_effect = FileNotFoundError - new_data = ['new data 1', 'new data 2'] - linep = lineparsermod.AppendLineParser('foo', 'bar') - linep.new_data = new_data - with linep.open(): - assert list(linep) == new_data - - def test_get_recent_none(self, tmpdir): - """Test get_recent with no data.""" - (tmpdir / 'file2').ensure() - linep = lineparsermod.AppendLineParser(str(tmpdir), 'file2') - assert linep.get_recent() == [] - - def test_get_recent_little(self, lineparser): - """Test get_recent with little data.""" - data = [e + '\n' for e in self.BASE_DATA] - assert lineparser.get_recent() == data - - def test_get_recent_much(self, lineparser): - """Test get_recent with much data.""" - size = 64 - new_data = ['new data {}'.format(i) for i in range(size)] - lineparser.new_data = new_data - lineparser.save() - data = os.linesep.join(self.BASE_DATA + new_data) + os.linesep - data = [e + '\n' for e in data[-size:].splitlines()] - assert lineparser.get_recent(size) == data From 9d4888a7729fcd6f573b23d081c12e2f4bf91de1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 15 Apr 2017 23:01:24 -0400 Subject: [PATCH 070/337] Optimize qute://history for SQL backend. The old implementation was looping through the whole history list, which for SQL was selecting every row in the database. The history benchmark was taking ~2s. If this is rewritten as a specialized SQL query, the benchmark takes ~10ms, an order of magnitude faster than the original non-SQL implementation. --- qutebrowser/browser/history.py | 20 ++++++- qutebrowser/browser/qutescheme.py | 76 +++------------------------ tests/unit/browser/test_qutescheme.py | 13 +++-- 3 files changed, 33 insertions(+), 76 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index d7753cd03..dc22d37a8 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -86,13 +86,31 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" - self.insert([entry.url_str(), entry.title, entry.atime, + self.insert([entry.url_str(), entry.title, int(entry.atime), entry.redirect]) def get_recent(self): """Get the most recent history entries.""" return self.select(sort_by='atime', sort_order='desc', limit=100) + def entries_between(self, earliest, latest): + """Iterate non-redirect, non-qute entries between two timestamps. + + Args: + earliest: Omit timestamps earlier than this. + latest: Omit timestamps later than this. + """ + result = sql.run_query('SELECT * FROM History ' + 'where not redirect ' + 'and not url like "qute://%" ' + 'and atime > {} ' + 'and atime <= {} ' + 'ORDER BY atime desc' + .format(earliest, latest)) + while result.next(): + rec = result.record() + yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + @cmdutils.register(name='history-clear', instance='web-history') def clear(self, force=False): """Clear all browsing history. diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index d12fb8f8e..55a16801a 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -186,81 +186,17 @@ def qute_bookmarks(_url): return 'text/html', html -def history_data(start_time): # noqa +def history_data(start_time): """Return history data Arguments: start_time -- select history starting from this timestamp. """ - def history_iter(start_time, reverse=False): - """Iterate through the history and get items we're interested. - - Arguments: - reverse -- whether to reverse the history_dict before iterating. - """ - history = list(objreg.get('web-history')) - if reverse: - history = reversed(history) - - # when history_dict is not reversed, we need to keep track of last item - # so that we can yield its atime - last_item = None - - # end is 24hrs earlier than start - end_time = start_time - 24*60*60 - - for item in history: - # Skip redirects - # Skip qute:// links - if item.redirect or item.url.startswith('qute://'): - continue - - # Skip items out of time window - item_newer = item.atime > start_time - item_older = item.atime <= end_time - if reverse: - # history_dict is reversed, we are going back in history. - # so: - # abort if item is older than start_time+24hr - # skip if item is newer than start - if item_older: - yield {"next": int(item.atime)} - return - if item_newer: - continue - else: - # history_dict isn't reversed, we are going forward in history. - # so: - # abort if item is newer than start_time - # skip if item is older than start_time+24hrs - if item_older: - last_item = item - continue - if item_newer: - yield {"next": int(last_item.atime if last_item else -1)} - return - - # Use item's url as title if there's no title. - item_title = item.title if item.title else item.url - item_time = int(item.atime * 1000) - - yield {"url": item.url, "title": item_title, "time": item_time} - - # if we reached here, we had reached the end of history - yield {"next": int(last_item.atime if last_item else -1)} - - if sys.hexversion >= 0x03050000: - # On Python >= 3.5 we can reverse the ordereddict in-place and thus - # apply an additional performance improvement in history_iter. - # On my machine, this gets us down from 550ms to 72us with 500k old - # items. - history = history_iter(start_time, reverse=True) - else: - # On Python 3.4, we can't do that, so we'd need to copy the entire - # history to a list. There, filter first and then reverse it here. - history = reversed(list(history_iter(start_time, reverse=False))) - - return list(history) + # end is 24hrs earlier than start + end_time = start_time - 24*60*60 + entries = objreg.get('web-history').entries_between(end_time, start_time) + return [{"url": e.url, "title": e.title or e.url, "time": e.atime * 1000} + for e in entries] @add_handler('history') diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 86a322322..0f9f43373 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -132,6 +132,7 @@ class TestHistoryHandler: assert item['time'] <= start_time * 1000 assert item['time'] > end_time * 1000 + @pytest.mark.skip("TODO: do we need next?") @pytest.mark.parametrize("start_time_offset, next_time", [ (0, 24*60*60), (24*60*60, 48*60*60), @@ -153,14 +154,16 @@ class TestHistoryHandler: assert items[0]["next"] == now - next_time def test_qute_history_benchmark(self, fake_web_history, benchmark, now): - # items must be earliest-first to ensure history is sorted properly - for t in range(100000, 0, -1): # one history per second - entry = history.Entry( + entries = [] + for t in range(100000): # one history per second + entry = fake_web_history.Entry( atime=str(now - t), url=QUrl('www.x.com/{}'.format(t)), - title='x at {}'.format(t)) - fake_web_history._add_entry(entry) + title='x at {}'.format(t), + redirect=False) + entries.append(entry) + fake_web_history.insert_batch(entries) url = QUrl("qute://history/data?start_time={}".format(now)) _mimetype, data = benchmark(qutescheme.qute_history, url) assert len(json.loads(data)) > 1 From 71191f10a28f86b692864bba6679d9525207c7eb Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 18 Apr 2017 07:48:11 -0400 Subject: [PATCH 071/337] Only complete most recent atime for url. The history completion query is extended to pick only the most recent item for a given url. The tests in test_models now check for ordering of elements. --- qutebrowser/browser/qutescheme.py | 2 +- qutebrowser/completion/models/sqlcategory.py | 6 +- qutebrowser/completion/models/urlmodel.py | 4 +- tests/helpers/fixtures.py | 2 +- tests/helpers/stubs.py | 18 --- tests/unit/completion/test_models.py | 121 ++++++++++++++----- tests/unit/completion/test_sqlcategory.py | 10 ++ 7 files changed, 107 insertions(+), 56 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 55a16801a..ba8486ff4 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -187,7 +187,7 @@ def qute_bookmarks(_url): def history_data(start_time): - """Return history data + """Return history data. Arguments: start_time -- select history starting from this timestamp. diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 693aaa841..9f78163ca 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -30,7 +30,7 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, sort_by=None, sort_order=None, select='*', - where=None, parent=None): + where=None, group_by=None, parent=None): """Create a new completion category backed by a sql table. Args: @@ -46,6 +46,7 @@ class SqlCategory(QSqlQueryModel): self._sort_order = sort_order self._select = select self._where = where + self._group_by = group_by self.set_pattern('', [0]) def set_pattern(self, pattern, columns_to_filter): @@ -67,6 +68,9 @@ class SqlCategory(QSqlQueryModel): if self._where: querystr += ' and ' + self._where + if self._group_by: + querystr += ' group by {}'.format(self._group_by) + if self._sort_by: assert self._sort_order in ['asc', 'desc'] querystr += ' order by {} {}'.format(self._sort_by, diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index ea3f3f610..9e601cd39 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -74,9 +74,9 @@ def url(): model.add_category(listcategory.ListCategory('Bookmarks', bookmarks)) timefmt = config.get('completion', 'timestamp-format') - select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) + select_time = "strftime('{}', max(atime), 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'History', sort_order='desc', sort_by='atime', + 'History', sort_order='desc', sort_by='atime', group_by='url', select='url, title, {}'.format(select_time), where='not redirect') model.add_category(hist_cat) return model diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 24087ed18..1a3dfd8ea 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -259,7 +259,7 @@ def bookmark_manager_stub(stubs): @pytest.fixture def session_manager_stub(stubs): - """Fixture which provides a fake web-history object.""" + """Fixture which provides a fake session-manager object.""" stub = stubs.SessionManagerStub() objreg.register('session-manager', stub) yield stub diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 8b2a235d3..05341103c 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -518,24 +518,6 @@ class QuickmarkManagerStub(UrlMarkManagerStub): self.delete(key) -class WebHistoryStub(QObject): - - """Stub for the web-history object.""" - - add_completion_item = pyqtSignal(history.Entry) - cleared = pyqtSignal() - - def __init__(self, parent=None): - super().__init__(parent) - self.history_dict = collections.OrderedDict() - - def __iter__(self): - return iter(self.history_dict.values()) - - def __len__(self): - return len(self.history_dict) - - class HostBlockerStub: """Stub for the host-blocker object.""" diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index e071f1584..cea9d0ccd 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -43,18 +43,18 @@ def _check_completions(model, expected): ... } """ + actual = {} assert model.rowCount() == len(expected) for i in range(0, model.rowCount()): catidx = model.index(i, 0) catname = model.data(catidx) - assert catname in expected - expected_cat = expected[catname] - assert model.rowCount(catidx) == len(expected_cat) + actual[catname] = [] for j in range(model.rowCount(catidx)): name = model.data(model.index(j, 0, parent=catidx)) desc = model.data(model.index(j, 1, parent=catidx)) misc = model.data(model.index(j, 2, parent=catidx)) - assert (name, desc, misc) in expected_cat + actual[catname].append((name, desc, misc)) + assert actual == expected # sanity-check the column_widths assert len(model.column_widths) == 3 assert sum(model.column_widths) == 100 @@ -155,18 +155,26 @@ def bookmarks(bookmark_manager_stub): @pytest.fixture -def web_history(stubs, init_sql): +def web_history_stub(stubs, init_sql): + return sql.SqlTable("History", ['url', 'title', 'atime', 'redirect']) + + +@pytest.fixture +def web_history(web_history_stub, init_sql): """Pre-populate the web-history database.""" - table = sql.SqlTable("History", ['url', 'title', 'atime', 'redirect']) - table.insert(['http://some-redirect.example.com', 'redirect', - datetime(2016, 9, 5).timestamp(), True]) - table.insert(['http://qutebrowser.org', 'qutebrowser', - datetime(2015, 9, 5).timestamp(), False]) - table.insert(['https://python.org', 'Welcome to Python.org', - datetime(2016, 3, 8).timestamp(), False]) - table.insert(['https://github.com', 'https://github.com', - datetime(2016, 5, 1).timestamp(), False]) - return table + web_history_stub.insert(['http://some-redirect.example.com', 'redirect', + datetime(2016, 9, 5).timestamp(), True]) + web_history_stub.insert(['http://qutebrowser.org', 'qutebrowser', + datetime(2015, 9, 5).timestamp(), False]) + web_history_stub.insert(['https://python.org', 'Welcome to Python.org', + datetime(2016, 2, 8).timestamp(), False]) + web_history_stub.insert(['https://python.org', 'Welcome to Python.org', + datetime(2016, 3, 8).timestamp(), False]) + web_history_stub.insert(['https://python.org', 'Welcome to Python.org', + datetime(2014, 3, 8).timestamp(), False]) + web_history_stub.insert(['https://github.com', 'https://github.com', + datetime(2016, 5, 1).timestamp(), False]) + return web_history_stub def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, @@ -186,15 +194,16 @@ def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, 'rr': 'roll', 'ro': 'rock'}) model = miscmodels.command() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Commands": [ - ('stop', 'stop qutebrowser', 's'), ('drop', 'drop all user data', ''), - ('roll', 'never gonna give you up', 'rr'), ('rock', "Alias for 'roll'", 'ro'), + ('roll', 'never gonna give you up', 'rr'), + ('stop', 'stop qutebrowser', 's'), ] }) @@ -214,23 +223,24 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): _patch_cmdutils(monkeypatch, stubs, module + '.cmdutils') _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') model = miscmodels.helptopic() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Commands": [ - (':stop', 'stop qutebrowser', 's'), (':drop', 'drop all user data', ''), - (':roll', 'never gonna give you up', 'rr'), (':hide', '', ''), + (':roll', 'never gonna give you up', 'rr'), + (':stop', 'stop qutebrowser', 's'), ], "Settings": [ ('general->time', 'Is an illusion.', None), ('general->volume', 'Goes to 11', None), + ('searchengines->DEFAULT', '', None), ('ui->gesture', 'Waggle your hands to control qutebrowser', None), ('ui->mind', 'Enable mind-control ui (experimental)', None), ('ui->voice', 'Whether to respond to voice commands', None), - ('searchengines->DEFAULT', '', None), ] }) @@ -238,6 +248,7 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub): def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -253,14 +264,15 @@ def test_quickmark_completion(qtmodeltester, quickmarks): def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Bookmarks": [ + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ('https://github.com', 'GitHub', None), ('https://python.org', 'Welcome to Python.org', None), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ] }) @@ -273,22 +285,24 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, - quickmarks, bookmarks, and urls are included - entries are sorted by access time - redirect entries are not included + - only the most recent entry is included for each url """ config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Quickmarks": [ - ('https://wiki.archlinux.org', 'aw', None), ('https://duckduckgo.com', 'ddg', None), + ('https://wiki.archlinux.org', 'aw', None), ('https://wikipedia.org', 'wiki', None), ], "Bookmarks": [ + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ('https://github.com', 'GitHub', None), ('https://python.org', 'Welcome to Python.org', None), - ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), ], "History": [ ('https://github.com', 'https://github.com', '2016-05-01'), @@ -298,17 +312,48 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, }) +@pytest.mark.parametrize('url, title, pattern, rowcount', [ + ('example.com', 'Site Title', '', 1), + ('example.com', 'Site Title', 'ex', 1), + ('example.com', 'Site Title', 'am', 1), + ('example.com', 'Site Title', 'com', 1), + ('example.com', 'Site Title', 'ex com', 1), + ('example.com', 'Site Title', 'com ex', 0), + ('example.com', 'Site Title', 'ex foo', 0), + ('example.com', 'Site Title', 'foo com', 0), + ('example.com', 'Site Title', 'exm', 0), + ('example.com', 'Site Title', 'Si Ti', 1), + ('example.com', 'Site Title', 'Ti Si', 0), + ('example.com', '', 'foo', 0), + ('foo_bar', '', '_', 1), + ('foobar', '', '_', 0), + ('foo%bar', '', '%', 1), + ('foobar', '', '%', 0), +]) +def test_url_completion_pattern(config_stub, web_history_stub, + quickmark_manager_stub, bookmark_manager_stub, + url, title, pattern, rowcount): + """Test that url completion filters by url and title.""" + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} + web_history_stub.insert([url, title, 0, False]) + model = urlmodel.url() + model.set_pattern(pattern) + # 2, 0 is History + assert model.rowCount(model.index(2, 0)) == rowcount + + def test_url_completion_delete_bookmark(qtmodeltester, config_stub, web_history, quickmarks, bookmarks, qtbot): """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - # delete item (1, 0) -> (bookmarks, 'https://github.com' ) - view = _mock_view_index(model, 1, 0, qtbot) + # delete item (1, 1) -> (bookmarks, 'https://github.com') + view = _mock_view_index(model, 1, 1, qtbot) model.delete_cur_item(view) assert 'https://github.com' not in bookmarks.marks assert 'https://python.org' in bookmarks.marks @@ -321,11 +366,12 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - # delete item (0, 1) -> (quickmarks, 'ddg' ) - view = _mock_view_index(model, 0, 1, qtbot) + # delete item (0, 0) -> (quickmarks, 'ddg' ) + view = _mock_view_index(model, 0, 0, qtbot) model.delete_cur_item(view) assert 'aw' in quickmarks.marks assert 'ddg' not in quickmarks.marks @@ -338,6 +384,7 @@ def test_url_completion_delete_history(qtmodeltester, config_stub, """Test that deleting a history entry is a noop.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -350,13 +397,14 @@ def test_url_completion_delete_history(qtmodeltester, config_stub, def test_session_completion(qtmodeltester, session_manager_stub): session_manager_stub.sessions = ['default', '1', '2'] model = miscmodels.session() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { - "Sessions": [('default', None, None), - ('1', None, None), - ('2', None, None)] + "Sessions": [('1', None, None), + ('2', None, None), + ('default', None, None)] }) @@ -371,6 +419,7 @@ def test_tab_completion(qtmodeltester, fake_web_tab, app_stub, win_registry, fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0), ] model = miscmodels.buffer() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -398,6 +447,7 @@ def test_tab_completion_delete(qtmodeltester, fake_web_tab, qtbot, app_stub, fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0), ] model = miscmodels.buffer() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -415,14 +465,15 @@ def test_setting_section_completion(qtmodeltester, monkeypatch, stubs): _patch_config_section_desc(monkeypatch, stubs, module + '.configdata.SECTION_DESC') model = configmodel.section() + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Sections": [ ('general', 'General/miscellaneous options.', None), - ('ui', 'General options related to the user interface.', None), ('searchengines', 'Definitions of search engines ...', None), + ('ui', 'General options related to the user interface.', None), ] }) @@ -435,6 +486,7 @@ def test_setting_option_completion(qtmodeltester, monkeypatch, stubs, 'mind': 'on', 'voice': 'sometimes'}} model = configmodel.option('ui') + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -457,6 +509,7 @@ def test_setting_option_completion_valuelist(qtmodeltester, monkeypatch, stubs, } } model = configmodel.option('searchengines') + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -471,6 +524,7 @@ def test_setting_value_completion(qtmodeltester, monkeypatch, stubs, _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') config_stub.data = {'general': {'volume': '0'}} model = configmodel.value('general', 'volume') + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -503,16 +557,17 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, 'rr': 'roll', 'ro': 'rock'}) model = miscmodels.bind('s') + model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) _check_completions(model, { "Commands": [ - ('stop', 'stop qutebrowser', 's'), ('drop', 'drop all user data', ''), ('hide', '', ''), - ('roll', 'never gonna give you up', 'rr'), ('rock', "Alias for 'roll'", 'ro'), + ('roll', 'never gonna give you up', 'rr'), + ('stop', 'stop qutebrowser', 's'), ] }) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 03288f016..f29589717 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -149,6 +149,16 @@ def test_where(): _validate(cat, [('foo', 'bar', False)]) +def test_group(): + table = sql.SqlTable('Foo', ['a', 'b']) + table.insert(['foo', 1]) + table.insert(['bar', 3]) + table.insert(['foo', 2]) + table.insert(['bar', 0]) + cat = sqlcategory.SqlCategory('Foo', select='a, max(b)', group_by='a') + _validate(cat, [('bar', 3), ('foo', 2)]) + + def test_entry(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) assert hasattr(table.Entry, 'a') From 0aa0478327246e68620f2326cefd9ca9ecc02785 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 7 May 2017 11:06:33 -0400 Subject: [PATCH 072/337] Use EXISTS for SqlTable.contains. --- qutebrowser/misc/sql.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 9cbf6024b..21af483c9 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -146,9 +146,10 @@ class SqlTable(QObject): field: Field to match. value: Value to check for the given field. """ - query = run_query("SELECT * FROM {} where {} = ? LIMIT 1" + query = run_query("Select EXISTS(SELECT * FROM {} where {} = ?)" .format(self._name, field), [value]) - return query.next() + query.next() + return query.value(0) def __len__(self): """Return the count of rows in the table.""" From eb6126906899e5d957bd55adf915468e2ab25c2f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 7 May 2017 22:21:46 -0400 Subject: [PATCH 073/337] Fix qute://history javascript for SQL. Returning "next" was no longer possible as the SQL query does not fetch more items than necessary. This is solved by using a start time, a limit, and an offset. The offset is needed to prevent fetching duplicate items if multiple entries have the same timestamp. Two of the history tests that relied on qute://history were changed to rely on qute://history/data instead to make them less failure-prone. --- qutebrowser/browser/history.py | 19 +++++++++++++ qutebrowser/browser/qutescheme.py | 26 ++++++++++++----- qutebrowser/javascript/history.js | 34 +++++++++++++++-------- tests/end2end/features/history.feature | 4 +-- tests/unit/browser/webkit/test_history.py | 27 ++++++++++++++++++ 5 files changed, 90 insertions(+), 20 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index dc22d37a8..2f7040c21 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -111,6 +111,25 @@ class WebHistory(sql.SqlTable): rec = result.record() yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + def entries_before(self, latest, limit, offset): + """Iterate non-redirect, non-qute entries occuring before a timestamp. + + Args: + latest: Omit timestamps more recent than this. + limit: Max number of entries to include. + offset: Number of entries to skip. + """ + result = sql.run_query('SELECT * FROM History ' + 'where not redirect ' + 'and not url like "qute://%" ' + 'and atime <= {} ' + 'ORDER BY atime desc ' + 'limit {} offset {}' + .format(latest, limit, offset)) + while result.next(): + rec = result.record() + yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + @cmdutils.register(name='history-clear', instance='web-history') def clear(self, force=False): """Clear all browsing history. diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index ba8486ff4..6039fe55e 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -186,15 +186,22 @@ def qute_bookmarks(_url): return 'text/html', html -def history_data(start_time): +def history_data(start_time, offset=None): """Return history data. Arguments: - start_time -- select history starting from this timestamp. + start_time: select history starting from this timestamp. + offset: number of items to skip """ - # end is 24hrs earlier than start - end_time = start_time - 24*60*60 - entries = objreg.get('web-history').entries_between(end_time, start_time) + hist = objreg.get('web-history') + if offset is not None: + # end is 24hrs earlier than start + entries = hist.entries_before(start_time, limit=1000, offset=offset) + else: + # end is 24hrs earlier than start + end_time = start_time - 24*60*60 + entries = hist.entries_between(end_time, start_time) + return [{"url": e.url, "title": e.title or e.url, "time": e.atime * 1000} for e in entries] @@ -203,6 +210,11 @@ def history_data(start_time): def qute_history(url): """Handler for qute://history. Display and serve history.""" if url.path() == '/data': + try: + offset = QUrlQuery(url).queryItemValue("offset") + offset = int(offset) if offset else None + except ValueError as e: + raise QuteSchemeError("Query parameter start_time is invalid", e) # Use start_time in query or current time. try: start_time = QUrlQuery(url).queryItemValue("start_time") @@ -210,7 +222,7 @@ def qute_history(url): except ValueError as e: raise QuteSchemeError("Query parameter start_time is invalid", e) - return 'text/html', json.dumps(history_data(start_time)) + return 'text/html', json.dumps(history_data(start_time, offset)) else: if ( config.get('content', 'allow-javascript') and @@ -244,7 +256,7 @@ def qute_history(url): (i["url"], i["title"], datetime.datetime.fromtimestamp(i["time"]/1000), QUrl(i["url"]).host()) - for i in history_data(start_time) if "next" not in i + for i in history_data(start_time) ] return 'text/html', jinja.render( diff --git a/qutebrowser/javascript/history.js b/qutebrowser/javascript/history.js index f46ceb49d..a6cf1d7aa 100644 --- a/qutebrowser/javascript/history.js +++ b/qutebrowser/javascript/history.js @@ -23,8 +23,12 @@ window.loadHistory = (function() { // Date of last seen item. var lastItemDate = null; - // The time to load next. + // Each request for new items includes the time of the last item and an + // offset. The offset is equal to the number of items from the previous + // request that had time=nextTime, and causes the next request to skip + // those items to avoid duplicates. var nextTime = null; + var nextOffset = 0; // The URL to fetch data from. var DATA_URL = "qute://history/data"; @@ -157,7 +161,15 @@ window.loadHistory = (function() { return; } - for (var i = 0, len = history.length - 1; i < len; i++) { + if (history.length == 0) { + // Reached end of history + window.onscroll = null; + EOF_MESSAGE.style.display = "block"; + LOAD_LINK.style.display = "none"; + return + } + + for (var i = 0, len = history.length; i < len; i++) { var item = history[i]; var currentItemDate = new Date(item.time); getSessionNode(currentItemDate).appendChild(makeHistoryRow( @@ -166,14 +178,12 @@ window.loadHistory = (function() { lastItemDate = currentItemDate; } - var next = history[history.length - 1].next; - if (next === -1) { - // Reached end of history - window.onscroll = null; - EOF_MESSAGE.style.display = "block"; - LOAD_LINK.style.display = "none"; - } else { - nextTime = next; + nextTime = history[history.length - 1].time + nextOffset = 0; + for (var i = history.length - 1; i >= 0; i--) { + if (history[i].time == nextTime) { + nextOffset++; + } } } @@ -183,9 +193,11 @@ window.loadHistory = (function() { */ function loadHistory() { if (nextTime === null) { - getJSON(DATA_URL, receiveHistory); + var url = DATA_URL.concat("?offset=0"); + getJSON(url, receiveHistory); } else { var url = DATA_URL.concat("?start_time=", nextTime.toString()); + var url = url.concat("&offset=", nextOffset.toString()); getJSON(url, receiveHistory); } } diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 3ef602712..6efd9ad0e 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -44,14 +44,14 @@ Feature: Page history Scenario: History with an error When I run :open file:///does/not/exist And I wait for "Error while loading file:///does/not/exist: Error opening /does/not/exist: *" in the log - And I open qute://history + And I open qute://history/data Then the page should contain the plaintext "Error loading page: file:///does/not/exist" @qtwebengine_todo: Error page message is not implemented Scenario: History with a 404 When I open status/404 without waiting And I wait for "Error while loading http://localhost:*/status/404: NOT FOUND" in the log - And I open qute://history + And I open qute://history/data Then the page should contain the plaintext "Error loading page: http://localhost:" And the page should contain the plaintext "/status/404" diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 4f1a3362d..2118a6c84 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -80,6 +80,33 @@ def test_get_recent(hist): ] +def test_entries_between(hist): + hist.add_url(QUrl('http://www.example.com/1'), atime=12345) + hist.add_url(QUrl('http://www.example.com/2'), atime=12346) + hist.add_url(QUrl('http://www.example.com/3'), atime=12347) + hist.add_url(QUrl('http://www.example.com/4'), atime=12348) + hist.add_url(QUrl('http://www.example.com/5'), atime=12348) + hist.add_url(QUrl('http://www.example.com/6'), atime=12349) + hist.add_url(QUrl('http://www.example.com/7'), atime=12350) + + times = [x.atime for x in hist.entries_between(12346, 12349)] + assert times == [12349, 12348, 12348, 12347] + + +def test_entries_before(hist): + hist.add_url(QUrl('http://www.example.com/1'), atime=12346) + hist.add_url(QUrl('http://www.example.com/2'), atime=12346) + hist.add_url(QUrl('http://www.example.com/3'), atime=12347) + hist.add_url(QUrl('http://www.example.com/4'), atime=12348) + hist.add_url(QUrl('http://www.example.com/5'), atime=12348) + hist.add_url(QUrl('http://www.example.com/6'), atime=12348) + hist.add_url(QUrl('http://www.example.com/7'), atime=12349) + hist.add_url(QUrl('http://www.example.com/8'), atime=12349) + + times = [x.atime for x in hist.entries_before(12348, limit=3, offset=2)] + assert times == [12348, 12347, 12346] + + def test_save(tmpdir, hist): hist.add_url(QUrl('http://example.com/'), atime=12345) hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) From 882da71397e30d932269687254cb4cd7b60a6a7c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 9 May 2017 08:01:31 -0400 Subject: [PATCH 074/337] Remove unused imports --- qutebrowser/browser/qutescheme.py | 1 - tests/helpers/stubs.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 6039fe55e..4ebc626d3 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -26,7 +26,6 @@ Module attributes: import json import os -import sys import time import urllib.parse import datetime diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 05341103c..b852426d2 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -29,7 +29,7 @@ from PyQt5.QtNetwork import (QNetworkRequest, QAbstractNetworkCache, QNetworkCacheMetaData) from PyQt5.QtWidgets import QCommonStyle, QLineEdit, QWidget, QTabBar -from qutebrowser.browser import browsertab, history +from qutebrowser.browser import browsertab from qutebrowser.config import configexc from qutebrowser.utils import usertypes, utils from qutebrowser.mainwindow import mainwindow From 119c33ac32afc4eabe21ecc2fc79b06730c91855 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 9 May 2017 12:02:59 -0400 Subject: [PATCH 075/337] Remove base.py from check_coverage. This module no longer exists. --- scripts/dev/check_coverage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index a2950b658..c01ae64af 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -156,8 +156,6 @@ PERFECT_FILES = [ ('tests/unit/utils/test_javascript.py', 'utils/javascript.py'), - ('tests/unit/completion/test_models.py', - 'completion/models/base.py'), ('tests/unit/completion/test_models.py', 'completion/models/urlmodel.py'), ('tests/unit/completion/test_sqlcategory.py', From 9b25b7ee5df8477e1d179aa2a81e6fdc6e1abca1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 9 May 2017 12:08:31 -0400 Subject: [PATCH 076/337] Fix misspelling of 'occurs' --- qutebrowser/browser/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 2f7040c21..89f850b9d 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -112,7 +112,7 @@ class WebHistory(sql.SqlTable): yield self.Entry(*[rec.value(i) for i in range(rec.count())]) def entries_before(self, latest, limit, offset): - """Iterate non-redirect, non-qute entries occuring before a timestamp. + """Iterate non-redirect, non-qute entries occurring before a timestamp. Args: latest: Omit timestamps more recent than this. From e201a42383891cc5142759536aabc72782a663c0 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 9 May 2017 21:18:15 -0400 Subject: [PATCH 077/337] Fix eslint errors --- qutebrowser/javascript/history.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/qutebrowser/javascript/history.js b/qutebrowser/javascript/history.js index a6cf1d7aa..3f5f9def6 100644 --- a/qutebrowser/javascript/history.js +++ b/qutebrowser/javascript/history.js @@ -161,14 +161,17 @@ window.loadHistory = (function() { return; } - if (history.length == 0) { + if (history.length === 0) { // Reached end of history window.onscroll = null; EOF_MESSAGE.style.display = "block"; LOAD_LINK.style.display = "none"; - return + return; } + nextTime = history[history.length - 1].time; + nextOffset = 0; + for (var i = 0, len = history.length; i < len; i++) { var item = history[i]; var currentItemDate = new Date(item.time); @@ -176,12 +179,7 @@ window.loadHistory = (function() { item.url, item.title, currentItemDate.toLocaleTimeString() )); lastItemDate = currentItemDate; - } - - nextTime = history[history.length - 1].time - nextOffset = 0; - for (var i = history.length - 1; i >= 0; i--) { - if (history[i].time == nextTime) { + if (item.time === nextTime) { nextOffset++; } } @@ -192,12 +190,11 @@ window.loadHistory = (function() { * @return {void} */ function loadHistory() { + var url = DATA_URL.concat("?offset=", nextOffset.toString()); if (nextTime === null) { - var url = DATA_URL.concat("?offset=0"); getJSON(url, receiveHistory); } else { - var url = DATA_URL.concat("?start_time=", nextTime.toString()); - var url = url.concat("&offset=", nextOffset.toString()); + url = url.concat("&start_time=", nextTime.toString()); getJSON(url, receiveHistory); } } From 87643040a41dd4c2a3af17d6d781a355b41cfa62 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 11 May 2017 08:20:37 -0400 Subject: [PATCH 078/337] Fix test_history for python < 3.6. Mock.assert_called is only in python 3.6. For earlier versions we must use `assert m.called`. Weird errors only appearing in CI, trying to debug... --- tests/unit/browser/webkit/test_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 2118a6c84..e6bbfcf69 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -122,7 +122,7 @@ def test_clear(qtbot, tmpdir, hist, mocker): m = mocker.patch('qutebrowser.browser.history.message.confirm_async') hist.clear() - m.assert_called() + assert m.called def test_clear_force(qtbot, tmpdir, hist): From 20000088dee5532c280fd741d665d3ab010f7a56 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 13 May 2017 21:32:47 -0400 Subject: [PATCH 079/337] Add debug-dump-history and fix sql history tests. Trying to read from the sql database from another process was flaky. This adds a debug-dump-history command which is used by the history BDD tests to validate the history contents. It outputs history in the old pre-SQL text format, so it might be useful for those who want to manipulate their history as text. --- qutebrowser/browser/history.py | 22 ++++++++++++ qutebrowser/misc/sql.py | 10 ++++-- tests/end2end/features/history.feature | 28 +++++++-------- tests/end2end/features/test_history_bdd.py | 41 +++++++++++----------- tests/unit/browser/webkit/test_history.py | 15 ++++++++ tests/unit/misc/test_sql.py | 3 +- 6 files changed, 79 insertions(+), 40 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 89f850b9d..e19463e75 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -19,6 +19,7 @@ """Simple history which gets written to disk.""" +import os import time from PyQt5.QtCore import pyqtSlot, QUrl @@ -233,6 +234,27 @@ class WebHistory(sql.SqlTable): .format(i, path, line)) self.insert_batch(rows) + @cmdutils.register(instance='web-history', debug=True) + def debug_dump_history(self, dest): + """Dump the history to a file in the old pre-SQL format. + + Args: + dest: Where to write the file to. + """ + dest = os.path.expanduser(dest) + + lines = ('{}{} {} {}' + .format(int(x.atime), '-r' * x.redirect, x.url, x.title) + for x in self.select(sort_by='atime', sort_order='asc')) + + with open(dest, 'w', encoding='utf-8') as f: + try: + f.write('\n'.join(lines)) + except OSError as e: + message.error('Could not write history: {}'.format(e)) + else: + message.info("Dumped history to {}.".format(dest)) + def init(parent=None): """Initialize the web history. diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 21af483c9..5c49767e8 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -200,8 +200,14 @@ class SqlTable(QObject): run_query("DELETE FROM {}".format(self._name)) self.changed.emit() - def select(self, sort_by, sort_order, limit): - """Remove all row from the table.""" + def select(self, sort_by, sort_order, limit=-1): + """Remove all row from the table. + + Args: + sort_by: name of column to sort by. + sort_order: 'asc' or 'desc'. + limit: max number of rows in result, defaults to -1 (unlimited). + """ result = run_query('SELECT * FROM {} ORDER BY {} {} LIMIT {}' .format(self._name, sort_by, sort_order, limit)) while result.next(): diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 6efd9ad0e..8174930f0 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -11,49 +11,45 @@ Feature: Page history Scenario: Simple history saving When I open data/numbers/1.txt And I open data/numbers/2.txt - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/numbers/1.txt http://localhost:(port)/data/numbers/2.txt Scenario: History item with title When I open data/title.html - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/title.html Test title Scenario: History item with redirect When I open redirect-to?url=data/title.html without waiting And I wait until data/title.html is loaded - Then the history file should contain: + Then the history should contain: r http://localhost:(port)/redirect-to?url=data/title.html Test title http://localhost:(port)/data/title.html Test title Scenario: History item with spaces in URL When I open data/title with spaces.html - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/title%20with%20spaces.html Test title Scenario: History item with umlauts When I open data/äöü.html - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/%C3%A4%C3%B6%C3%BC.html Chäschüechli - # The following two tests use qute://history instead of checking the - # history file due to a race condition with sqlite. - # https://github.com/qutebrowser/qutebrowser/pull/2295#issuecomment-292786138 @flaky @qtwebengine_todo: Error page message is not implemented Scenario: History with an error When I run :open file:///does/not/exist And I wait for "Error while loading file:///does/not/exist: Error opening /does/not/exist: *" in the log - And I open qute://history/data - Then the page should contain the plaintext "Error loading page: file:///does/not/exist" + Then the history should contain: + file:///does/not/exist Error loading page: file:///does/not/exist @qtwebengine_todo: Error page message is not implemented Scenario: History with a 404 When I open status/404 without waiting And I wait for "Error while loading http://localhost:*/status/404: NOT FOUND" in the log - And I open qute://history/data - Then the page should contain the plaintext "Error loading page: http://localhost:" - And the page should contain the plaintext "/status/404" + Then the history should contain: + http://localhost:(port)/status/404 Error loading page: http://localhost:(port)/status/404 Scenario: History with invalid URL When I run :tab-only @@ -78,19 +74,19 @@ Feature: Page history Scenario: Clearing history When I open data/title.html And I run :history-clear --force - Then the history file should be empty + Then the history should be empty Scenario: Clearing history with confirmation When I open data/title.html And I run :history-clear And I wait for "Asking question <* title='Clear all browsing history?'>, *" in the log And I run :prompt-accept yes - Then the history file should be empty + Then the history should be empty Scenario: History with yanked URL and 'add to history' flag When I open data/hints/html/simple.html And I hint with args "--add-history links yank" and follow a - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/hints/html/simple.html Simple link http://localhost:(port)/data/hello.txt diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 70f23f86b..197bc3d20 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -17,34 +17,33 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . +import logging import os.path +import re +import tempfile import pytest_bdd as bdd -from PyQt5.QtSql import QSqlDatabase - bdd.scenarios('history.feature') -@bdd.then(bdd.parsers.parse("the history file should contain:\n{expected}")) -def check_history(quteproc, httpbin, expected): - path = os.path.join(quteproc.basedir, 'data', 'history.sqlite') - db = QSqlDatabase.addDatabase('QSQLITE') - db.setDatabaseName(path) - assert db.open(), 'Failed to open history database' - query = db.exec_('select * from History') - actual = [] - while query.next(): - rec = query.record() - url = rec.value(0) - title = rec.value(1) - redirect = rec.value(3) - actual.append('{} {} {}'.format('r' * redirect, url, title).strip()) - db = None - QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName()) - assert actual == expected.replace('(port)', str(httpbin.port)).splitlines() +@bdd.then(bdd.parsers.parse("the history should contain:\n{expected}")) +def check_history(quteproc, expected, httpbin): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, 'history') + quteproc.send_cmd(':debug-dump-history "{}"'.format(path)) + quteproc.wait_for(category='message', loglevel=logging.INFO, + message='Dumped history to {}.'.format(path)) + + with open(path, 'r', encoding='utf-8') as f: + # ignore access times, they will differ in each run + actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() + for line in f.read().splitlines()) + + expected = expected.replace('(port)', str(httpbin.port)) + assert actual == expected -@bdd.then("the history file should be empty") +@bdd.then("the history should be empty") def check_history_empty(quteproc, httpbin): - check_history(quteproc, httpbin, '') + check_history(quteproc, '', httpbin) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index e6bbfcf69..16b86ca4a 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -261,3 +261,18 @@ def test_read_invalid(hist, tmpdir, line): with pytest.raises(Exception): hist.read(str(histfile)) + + +def test_debug_dump_history(hist, tmpdir): + hist.add_url(QUrl('http://example.com/1'), title="Title1", atime=12345) + hist.add_url(QUrl('http://example.com/2'), title="Title2", atime=12346) + hist.add_url(QUrl('http://example.com/3'), title="Title3", atime=12347) + hist.add_url(QUrl('http://example.com/4'), title="Title4", atime=12348, + redirect=True) + histfile = tmpdir / 'history' + hist.debug_dump_history(str(histfile)) + expected = ['12345 http://example.com/1 Title1', + '12346 http://example.com/2 Title2', + '12347 http://example.com/3 Title3', + '12348-r http://example.com/4 Title4'] + assert histfile.read() == '\n'.join(expected) diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index edc156a55..ca736b0ad 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -53,7 +53,8 @@ def test_iter(): @pytest.mark.parametrize('rows, sort_by, sort_order, limit, result', [ ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', 5, [(1, 6), (2, 5), (3, 4)]), ([[2, 5], [1, 6], [3, 4]], 'a', 'desc', 3, [(3, 4), (2, 5), (1, 6)]), - ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]) + ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]), + ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', -1, [(1, 6), (2, 5), (3, 4)]), ]) def test_select(rows, sort_by, sort_order, limit, result): table = sql.SqlTable('Foo', ['a', 'b']) From e67da51662071a63c8dffa1916144944a41def20 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 23 May 2017 09:01:20 -0400 Subject: [PATCH 080/337] Use prepared SQL queries. --- qutebrowser/browser/history.py | 37 +++--- qutebrowser/completion/models/sqlcategory.py | 54 ++++---- qutebrowser/completion/models/urlmodel.py | 1 + qutebrowser/misc/sql.py | 132 +++++++++---------- tests/unit/completion/test_sqlcategory.py | 12 +- 5 files changed, 115 insertions(+), 121 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e19463e75..d8b1f81d5 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -78,6 +78,19 @@ class WebHistory(sql.SqlTable): def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) + self._between_query = sql.Query('SELECT * FROM History ' + 'where not redirect ' + 'and not url like "qute://%" ' + 'and atime > ? ' + 'and atime <= ? ' + 'ORDER BY atime desc') + + self._before_query = sql.Query('SELECT * FROM History ' + 'where not redirect ' + 'and not url like "qute://%" ' + 'and atime <= ? ' + 'ORDER BY atime desc ' + 'limit ? offset ?') def __repr__(self): return utils.get_repr(self, length=len(self)) @@ -101,16 +114,8 @@ class WebHistory(sql.SqlTable): earliest: Omit timestamps earlier than this. latest: Omit timestamps later than this. """ - result = sql.run_query('SELECT * FROM History ' - 'where not redirect ' - 'and not url like "qute://%" ' - 'and atime > {} ' - 'and atime <= {} ' - 'ORDER BY atime desc' - .format(earliest, latest)) - while result.next(): - rec = result.record() - yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + self._between_query.run([earliest, latest]) + return iter(self._between_query) def entries_before(self, latest, limit, offset): """Iterate non-redirect, non-qute entries occurring before a timestamp. @@ -120,16 +125,8 @@ class WebHistory(sql.SqlTable): limit: Max number of entries to include. offset: Number of entries to skip. """ - result = sql.run_query('SELECT * FROM History ' - 'where not redirect ' - 'and not url like "qute://%" ' - 'and atime <= {} ' - 'ORDER BY atime desc ' - 'limit {} offset {}' - .format(latest, limit, offset)) - while result.next(): - rec = result.record() - yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + self._before_query.run([latest, limit, offset]) + return iter(self._before_query) @cmdutils.register(name='history-clear', instance='web-history') def clear(self, force=False): diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 9f78163ca..543ea2ade 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -29,12 +29,13 @@ from qutebrowser.misc import sql class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" - def __init__(self, name, *, sort_by=None, sort_order=None, select='*', - where=None, group_by=None, parent=None): + def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, + select='*', where=None, group_by=None, parent=None): """Create a new completion category backed by a sql table. Args: name: Name of category, and the table in the database. + filter_fields: Names of fields to apply filter pattern to. select: A custom result column expression for the select statement. where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. @@ -42,11 +43,24 @@ class SqlCategory(QSqlQueryModel): """ super().__init__(parent=parent) self.name = name - self._sort_by = sort_by - self._sort_order = sort_order - self._select = select - self._where = where - self._group_by = group_by + + querystr = 'select {} from {} where ('.format(select, name) + # the incoming pattern will have literal % and _ escaped with '\' + # we need to tell sql to treat '\' as an escape character + querystr += ' or '.join("{} like ? escape '\\'".format(f) + for f in filter_fields) + querystr += ')' + + if where: + querystr += ' and ' + where + if group_by: + querystr += ' group by {}'.format(group_by) + if sort_by: + assert sort_order in ['asc', 'desc'] + querystr += ' order by {} {}'.format(sort_by, sort_order) + + self._query = sql.Query(querystr) + self._param_count=len(filter_fields) self.set_pattern('', [0]) def set_pattern(self, pattern, columns_to_filter): @@ -56,31 +70,13 @@ class SqlCategory(QSqlQueryModel): pattern: string pattern to filter by. columns_to_filter: indices of columns to apply pattern to. """ - query = sql.run_query('select * from {} limit 1'.format(self.name)) - fields = [query.record().fieldName(i) for i in columns_to_filter] - - querystr = 'select {} from {} where ('.format(self._select, self.name) - # the incoming pattern will have literal % and _ escaped with '\' - # we need to tell sql to treat '\' as an escape character - querystr += ' or '.join("{} like ? escape '\\'".format(f) - for f in fields) - querystr += ')' - if self._where: - querystr += ' and ' + self._where - - if self._group_by: - querystr += ' group by {}'.format(self._group_by) - - if self._sort_by: - assert self._sort_order in ['asc', 'desc'] - querystr += ' order by {} {}'.format(self._sort_by, - self._sort_order) - + # TODO: eliminate columns_to_filter + #assert len(columns_to_filter) == self._param_count # escape to treat a user input % or _ as a literal, not a wildcard pattern = pattern.replace('%', '\\%') pattern = pattern.replace('_', '\\_') # treat spaces as wildcards to match any of the typed words pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) - query = sql.run_query(querystr, [pattern for _ in fields]) - self.setQuery(query) + self._query.run([pattern] * self._param_count) + self.setQuery(self._query) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 9e601cd39..29ecdca9f 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -77,6 +77,7 @@ def url(): select_time = "strftime('{}', max(atime), 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( 'History', sort_order='desc', sort_by='atime', group_by='url', + filter_fields=['url', 'title'], select='url, title, {}'.format(select_time), where='not redirect') model.add_category(hist_cat) return model diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 5c49767e8..5acc620fb 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -50,56 +50,45 @@ def close(): def version(): """Return the sqlite version string.""" - result = run_query("select sqlite_version()") - result.next() - return result.record().value(0) + q = Query("select sqlite_version()") + q.run() + return q.value() -def _prepare_query(querystr): - log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) - database = QSqlDatabase.database() - query = QSqlQuery(database) - query.prepare(querystr) - return query +class Query(QSqlQuery): + """A prepared SQL Query.""" -def run_query(querystr, values=None): - """Run the given SQL query string on the database. + def __init__(self, querystr): + super().__init__(QSqlDatabase.database()) + log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) + self.prepare(querystr) - Args: - values: A list of positional parameter bindings. - """ - query = _prepare_query(querystr) - for val in values or []: - query.addBindValue(val) - log.sql.debug('Query bindings: {}'.format(query.boundValues())) - if not query.exec_(): - raise SqlException('Failed to exec query "{}": "{}"'.format( - querystr, query.lastError().text())) - return query + def __iter__(self): + assert self.isActive(), "Cannot iterate inactive query" + rec = self.record() + fields = [rec.fieldName(i) for i in range(rec.count())] + rowtype = collections.namedtuple('ResultRow', fields) + while self.next(): + rec = self.record() + yield rowtype(*[rec.value(i) for i in range(rec.count())]) -def run_batch(querystr, values): - """Run the given SQL query string on the database in batch mode. + def run(self, values=None): + """Execute the prepared query.""" + log.sql.debug('Running SQL query: "{}"'.format(self.lastQuery())) + for val in values or []: + self.addBindValue(val) + log.sql.debug('self bindings: {}'.format(self.boundValues())) + if not self.exec_(): + raise SqlException('Failed to exec self "{}": "{}"'.format( + self.lastself(), self.lastError().text())) - Args: - values: A list of lists, where each inner list contains positional - bindings for one run of the batch. - """ - query = _prepare_query(querystr) - transposed = [list(row) for row in zip(*values)] - for val in transposed: - query.addBindValue(val) - log.sql.debug('Batch Query bindings: {}'.format(query.boundValues())) - - db = QSqlDatabase.database() - db.transaction() - if not query.execBatch(): - raise SqlException('Failed to exec query "{}": "{}"'.format( - querystr, query.lastError().text())) - db.commit() - - return query + def value(self): + """Return the result of a single-value query (e.g. an EXISTS).""" + ok = self.next() + assert ok, "No result for single-result query" + return self.record().value(0) class SqlTable(QObject): @@ -127,17 +116,17 @@ class SqlTable(QObject): """ super().__init__(parent) self._name = name - run_query("CREATE TABLE IF NOT EXISTS {} ({})" + q = Query("CREATE TABLE IF NOT EXISTS {} ({})" .format(name, ','.join(fields))) + q.run() # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) def __iter__(self): """Iterate rows in the table.""" - result = run_query("SELECT * FROM {}".format(self._name)) - while result.next(): - rec = result.record() - yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + q = Query("SELECT * FROM {}".format(self._name)) + q.run() + return iter(q) def contains(self, field, value): """Return whether the table contains the matching item. @@ -146,16 +135,16 @@ class SqlTable(QObject): field: Field to match. value: Value to check for the given field. """ - query = run_query("Select EXISTS(SELECT * FROM {} where {} = ?)" - .format(self._name, field), [value]) - query.next() - return query.value(0) + q = Query("Select EXISTS(SELECT * FROM {} where {} = ?)" + .format(self._name, field)) + q.run([value]) + return q.value() def __len__(self): """Return the count of rows in the table.""" - result = run_query("SELECT count(*) FROM {}".format(self._name)) - result.next() - return result.value(0) + q = Query("SELECT count(*) FROM {}".format(self._name)) + q.run() + return q.value() def delete(self, value, field): """Remove all rows for which `field` equals `value`. @@ -167,9 +156,9 @@ class SqlTable(QObject): Return: The number of rows deleted. """ - query = run_query("DELETE FROM {} where {} = ?".format( - self._name, field), [value]) - if not query.numRowsAffected(): + q = Query("DELETE FROM {} where {} = ?".format(self._name, field)) + q.run([value]) + if not q.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() @@ -180,8 +169,8 @@ class SqlTable(QObject): values: A list of values to insert. """ paramstr = ','.join(['?'] * len(values)) - run_query("INSERT INTO {} values({})".format(self._name, paramstr), - values) + q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q.run(values) self.changed.emit() def insert_batch(self, rows): @@ -191,13 +180,23 @@ class SqlTable(QObject): rows: A list of lists, where each sub-list is a row. """ paramstr = ','.join(['?'] * len(rows[0])) - run_batch("INSERT INTO {} values({})".format(self._name, paramstr), - rows) + q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + + transposed = [list(row) for row in zip(*rows)] + for val in transposed: + q.addBindValue(val) + + db = QSqlDatabase.database() + db.transaction() + if not q.execBatch(): + raise SqlException('Failed to exec query "{}": "{}"'.format( + q.lastQuery(), q.lastError().text())) + db.commit() self.changed.emit() def delete_all(self): """Remove all row from the table.""" - run_query("DELETE FROM {}".format(self._name)) + Query("DELETE FROM {}".format(self._name)).run() self.changed.emit() def select(self, sort_by, sort_order, limit=-1): @@ -208,8 +207,7 @@ class SqlTable(QObject): sort_order: 'asc' or 'desc'. limit: max number of rows in result, defaults to -1 (unlimited). """ - result = run_query('SELECT * FROM {} ORDER BY {} {} LIMIT {}' - .format(self._name, sort_by, sort_order, limit)) - while result.next(): - rec = result.record() - yield self.Entry(*[rec.value(i) for i in range(rec.count())]) + q = Query('SELECT * FROM {} ORDER BY {} {} LIMIT ?' + .format(self._name, sort_by, sort_order)) + q.run([limit]) + return q diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index f29589717..3d0b07d07 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -74,7 +74,7 @@ def test_sorting(sort_by, sort_order, data, expected): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: table.insert(row) - cat = sqlcategory.SqlCategory('Foo', sort_by=sort_by, + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) _validate(cat, expected) @@ -129,7 +129,8 @@ def test_set_pattern(pattern, filter_cols, before, after): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: table.insert(row) - cat = sqlcategory.SqlCategory('Foo') + filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] + cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) cat.set_pattern(pattern, filter_cols) _validate(cat, after) @@ -137,7 +138,7 @@ def test_set_pattern(pattern, filter_cols, before, after): def test_select(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert(['foo', 'bar', 'baz']) - cat = sqlcategory.SqlCategory('Foo', select='b, c, a') + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') _validate(cat, [('bar', 'baz', 'foo')]) @@ -145,7 +146,7 @@ def test_where(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert(['foo', 'bar', False]) table.insert(['baz', 'biz', True]) - cat = sqlcategory.SqlCategory('Foo', where='not c') + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') _validate(cat, [('foo', 'bar', False)]) @@ -155,7 +156,8 @@ def test_group(): table.insert(['bar', 3]) table.insert(['foo', 2]) table.insert(['bar', 0]) - cat = sqlcategory.SqlCategory('Foo', select='a, max(b)', group_by='a') + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], + select='a, max(b)', group_by='a') _validate(cat, [('bar', 3), ('foo', 2)]) From b1b521e0c2ad814bf3eda76b69c47560bf027c12 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 23 May 2017 22:20:25 -0400 Subject: [PATCH 081/337] Fix two history tests added recently. These were added on master and needed to be adjusted slightly for the new history check (which doesn't rely on reading a history file anymore). --- tests/end2end/features/history.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 8174930f0..9317bde64 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -61,14 +61,14 @@ Feature: Page history When I open data/data_link.html And I run :click-element id link And I wait until data:;base64,cXV0ZWJyb3dzZXI= is loaded - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/data_link.html data: link Scenario: History with view-source URL When I open data/title.html And I run :view-source And I wait for "Changing title for idx * to 'Source for http://localhost:*/data/title.html'" in the log - Then the history file should contain: + Then the history should contain: http://localhost:(port)/data/title.html Test title Scenario: Clearing history From 39b561a1821689a933ca31a64b2ece47e03adff5 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 24 May 2017 07:38:24 -0400 Subject: [PATCH 082/337] Fix BaseLineParser::test_double_open. Don't tie the test to a particular error message. Ths failed because a typo was fixed (AppendLineParser -> LineParser). --- tests/unit/misc/test_lineparser.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/misc/test_lineparser.py b/tests/unit/misc/test_lineparser.py index adae2485d..dcdb724cd 100644 --- a/tests/unit/misc/test_lineparser.py +++ b/tests/unit/misc/test_lineparser.py @@ -58,8 +58,7 @@ class TestBaseLineParser: mocker.patch('builtins.open', mock.mock_open()) with lineparser._open('r'): - with pytest.raises(IOError, match="Refusing to double-open " - "AppendLineParser."): + with pytest.raises(IOError): with lineparser._open('r'): pass From 2c501f7fb712289656da5895262d4bf1338b21d4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 24 May 2017 12:44:57 -0400 Subject: [PATCH 083/337] Fix url completion benchmark. Still had old code from pre-SQL era. --- tests/unit/completion/test_models.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index cea9d0ccd..8666ecc5b 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -580,21 +580,21 @@ def test_url_completion_benchmark(benchmark, config_stub, config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 1000} - entries = [history.Entry( + entries = [web_history_stub.Entry( atime=i, - url=QUrl('http://example.com/{}'.format(i)), - title='title{}'.format(i)) + url='http://example.com/{}'.format(i), + title='title{}'.format(i), + redirect=False) for i in range(100000)] - web_history_stub.history_dict = collections.OrderedDict( - ((e.url_str(), e) for e in entries)) + web_history_stub.insert_batch(entries) quickmark_manager_stub.marks = collections.OrderedDict( - (e.title, e.url_str()) + (e.title, e.url) for e in entries[0:1000]) bookmark_manager_stub.marks = collections.OrderedDict( - (e.url_str(), e.title) + (e.url, e.title) for e in entries[0:1000]) def bench(): From cf89ffa9710afacb0b928dadda74774ea42f3ce6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 25 May 2017 07:27:14 -0400 Subject: [PATCH 084/337] Fix pylint/flake8 errors --- qutebrowser/completion/models/sqlcategory.py | 4 ++-- qutebrowser/misc/sql.py | 4 ++-- tests/unit/completion/test_completionmodel.py | 2 +- tests/unit/completion/test_models.py | 1 - tests/unit/utils/test_version.py | 1 + 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 543ea2ade..5b8cf4482 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -60,10 +60,10 @@ class SqlCategory(QSqlQueryModel): querystr += ' order by {} {}'.format(sort_by, sort_order) self._query = sql.Query(querystr) - self._param_count=len(filter_fields) + self._param_count = len(filter_fields) self.set_pattern('', [0]) - def set_pattern(self, pattern, columns_to_filter): + def set_pattern(self, pattern, _columns_to_filter): """Set the pattern used to filter results. Args: diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 5acc620fb..0b81fc0f9 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -19,13 +19,13 @@ """Provides access to an in-memory sqlite database.""" +import collections + from PyQt5.QtCore import QObject, pyqtSignal from PyQt5.QtSql import QSqlDatabase, QSqlQuery from qutebrowser.utils import log -import collections - class SqlException(Exception): diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 05af902a3..d3cd23f68 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -19,8 +19,8 @@ """Tests for CompletionModel.""" -import hypothesis from unittest import mock +import hypothesis from hypothesis import strategies from qutebrowser.completion.models import completionmodel diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 8666ecc5b..c46287926 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -26,7 +26,6 @@ import pytest from PyQt5.QtCore import QUrl from PyQt5.QtWidgets import QTreeView -from qutebrowser.browser import history from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value from qutebrowser.misc import sql diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index c0c731f88..11bf326fb 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -798,6 +798,7 @@ def test_chromium_version_unpatched(qapp): assert version._chromium_version() not in ['', 'unknown', 'unavailable'] +# pylint: disable=too-many-locals @pytest.mark.parametrize(['git_commit', 'frozen', 'style', 'with_webkit', 'known_distribution'], [ (True, False, True, True, True), # normal From 8fb6f45bec9c82d26a653a1e96c6701fad12b020 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 25 May 2017 12:59:05 -0400 Subject: [PATCH 085/337] Don't set pattern in SqlCategory constructor. This will be called by the Completer after construction anyways, this was a duplicate call that could be expensive. --- qutebrowser/completion/models/sqlcategory.py | 3 +-- tests/unit/completion/test_sqlcategory.py | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 5b8cf4482..56d7ec399 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -61,9 +61,8 @@ class SqlCategory(QSqlQueryModel): self._query = sql.Query(querystr) self._param_count = len(filter_fields) - self.set_pattern('', [0]) - def set_pattern(self, pattern, _columns_to_filter): + def set_pattern(self, pattern, _columns_to_filter=None): """Set the pattern used to filter results. Args: diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 3d0b07d07..2f3b8474d 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -76,6 +76,7 @@ def test_sorting(sort_by, sort_order, data, expected): table.insert(row) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) + cat.set_pattern('') _validate(cat, expected) @@ -139,6 +140,7 @@ def test_select(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert(['foo', 'bar', 'baz']) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') + cat.set_pattern('') _validate(cat, [('bar', 'baz', 'foo')]) @@ -147,6 +149,7 @@ def test_where(): table.insert(['foo', 'bar', False]) table.insert(['baz', 'biz', True]) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') + cat.set_pattern('') _validate(cat, [('foo', 'bar', False)]) @@ -158,6 +161,7 @@ def test_group(): table.insert(['bar', 0]) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='a, max(b)', group_by='a') + cat.set_pattern('') _validate(cat, [('bar', 3), ('foo', 2)]) From 2cd02be7b108550b204095512be03d5eaff148c8 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 29 May 2017 22:58:11 -0400 Subject: [PATCH 086/337] Remove CompletionModel.columns_to_filter. Instead set this on inidividual categories, as that is where it actually gets used. This makes it easier for SqlCompletionCategory to reuse a prepared query (as it gets the filter field names in its constructor). --- qutebrowser/completion/completiondelegate.py | 2 +- .../completion/models/completionmodel.py | 19 +++++++++++++------ qutebrowser/completion/models/listcategory.py | 7 +++---- qutebrowser/completion/models/miscmodels.py | 7 ++++--- qutebrowser/completion/models/sqlcategory.py | 5 ++++- qutebrowser/completion/models/urlmodel.py | 7 ++++--- tests/unit/completion/test_completionmodel.py | 5 ++--- tests/unit/completion/test_listcategory.py | 5 +++-- tests/unit/completion/test_sqlcategory.py | 2 +- 9 files changed, 35 insertions(+), 24 deletions(-) diff --git a/qutebrowser/completion/completiondelegate.py b/qutebrowser/completion/completiondelegate.py index 776b2164c..5813d6dcb 100644 --- a/qutebrowser/completion/completiondelegate.py +++ b/qutebrowser/completion/completiondelegate.py @@ -197,7 +197,7 @@ class CompletionItemDelegate(QStyledItemDelegate): if index.parent().isValid(): pattern = index.model().pattern - columns_to_filter = index.model().columns_to_filter + columns_to_filter = index.model().columns_to_filter(index) if index.column() in columns_to_filter and pattern: repl = r'\g<0>' text = re.sub(re.escape(pattern).replace(r'\ ', r'|'), diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 0ee81fdf5..9543792ed 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -34,15 +34,13 @@ class CompletionModel(QAbstractItemModel): Attributes: column_widths: The width percentages of the columns used in the completion view. - columns_to_filter: A list of indices of columns to apply the filter to. pattern: Current filter pattern, used for highlighting. _categories: The sub-categories. """ - def __init__(self, *, column_widths=(30, 70, 0), columns_to_filter=None, + def __init__(self, *, column_widths=(30, 70, 0), delete_cur_item=None, parent=None): super().__init__(parent) - self.columns_to_filter = columns_to_filter or [0] self.column_widths = column_widths self._categories = [] self.pattern = '' @@ -173,8 +171,6 @@ class CompletionModel(QAbstractItemModel): def set_pattern(self, pattern): """Set the filter pattern for all categories. - This will apply to the fields indicated in columns_to_filter. - Args: pattern: The filter pattern to set. """ @@ -182,7 +178,7 @@ class CompletionModel(QAbstractItemModel): # TODO: should pattern be saved in the view layer instead? self.pattern = pattern for cat in self._categories: - cat.set_pattern(pattern, self.columns_to_filter) + cat.set_pattern(pattern) def first_item(self): """Return the index of the first child (non-category) in the model.""" @@ -204,3 +200,14 @@ class CompletionModel(QAbstractItemModel): qtutils.ensure_valid(index) return index return QModelIndex() + + def columns_to_filter(self, index): + """Return the column indices the filter pattern applies to. + + Args: + index: index of the item to check. + + Return: A list of integers. + """ + cat = self._cat_from_idx(index.parent()) + return cat.columns_to_filter if cat else [] diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index d24fd74d0..fbad3b063 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -35,25 +35,24 @@ class ListCategory(QSortFilterProxyModel): """Expose a list of items as a category for the CompletionModel.""" - def __init__(self, name, items, parent=None): + def __init__(self, name, items, columns_to_filter=None, parent=None): super().__init__(parent) self.name = name self.srcmodel = QStandardItemModel(parent=self) self.pattern = '' self.pattern_re = None - self.columns_to_filter = None + self.columns_to_filter = columns_to_filter or [0] for item in items: self.srcmodel.appendRow([QStandardItem(x) for x in item]) self.setSourceModel(self.srcmodel) - def set_pattern(self, val, columns_to_filter): + def set_pattern(self, val): """Setter for pattern. Args: val: The value to set. """ with debug.log_time(log.completion, 'Setting filter pattern'): - self.columns_to_filter = columns_to_filter self.pattern = val val = re.sub(r' +', r' ', val) # See #1919 val = re.escape(val) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 368e1e6c6..62818f767 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -110,8 +110,7 @@ def buffer(): model = completionmodel.CompletionModel( column_widths=(6, 40, 54), - delete_cur_item=delete_buffer, - columns_to_filter=[idx_column, url_column, text_column]) + delete_cur_item=delete_buffer) for win_id in objreg.window_registry: tabbed_browser = objreg.get('tabbed-browser', scope='window', @@ -124,8 +123,10 @@ def buffer(): tabs.append(("{}/{}".format(win_id, idx + 1), tab.url().toDisplayString(), tabbed_browser.page_title(idx))) - cat = listcategory.ListCategory("{}".format(win_id), tabs) + cat = listcategory.ListCategory("{}".format(win_id), tabs, + columns_to_filter=[idx_column, url_column, text_column]) model.add_category(cat) + return model diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 56d7ec399..b8cf07ffa 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -61,8 +61,11 @@ class SqlCategory(QSqlQueryModel): self._query = sql.Query(querystr) self._param_count = len(filter_fields) + rec = self._query.record() + # will this work? + self.columns_to_filter = [rec.indexOf(n) for n in filter_fields] - def set_pattern(self, pattern, _columns_to_filter=None): + def set_pattern(self, pattern): """Set the pattern used to filter results. Args: diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 29ecdca9f..9e2571848 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -63,15 +63,16 @@ def url(): """ model = completionmodel.CompletionModel( column_widths=(40, 50, 10), - columns_to_filter=[_URLCOL, _TEXTCOL], delete_cur_item=_delete_url) quickmarks = ((url, name) for (name, url) in objreg.get('quickmark-manager').marks.items()) bookmarks = objreg.get('bookmark-manager').marks.items() - model.add_category(listcategory.ListCategory('Quickmarks', quickmarks)) - model.add_category(listcategory.ListCategory('Bookmarks', bookmarks)) + model.add_category(listcategory.ListCategory('Quickmarks', quickmarks, + columns_to_filter=[0, 1])) + model.add_category(listcategory.ListCategory('Bookmarks', bookmarks, + columns_to_filter=[0, 1])) timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', max(atime), 'unixepoch')".format(timefmt) diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index d3cd23f68..8d43f2883 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -65,11 +65,10 @@ def test_count(counts): @hypothesis.given(strategies.text()) def test_set_pattern(pat): """Validate the filtering and sorting results of set_pattern.""" - cols = [1, 2, 3] - model = completionmodel.CompletionModel(columns_to_filter=cols) + model = completionmodel.CompletionModel() cats = [mock.Mock(spec=['set_pattern'])] * 3 for c in cats: c.set_pattern = mock.Mock() model.add_category(c) model.set_pattern(pat) - assert all(c.set_pattern.called_with([pat, cols]) for c in cats) + assert all(c.set_pattern.called_with([pat]) for c in cats) diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index 0b4426bdb..a0c300473 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -65,6 +65,7 @@ def _validate(cat, expected): ]) def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" - cat = listcategory.ListCategory('Foo', before) - cat.set_pattern(pattern, filter_cols) + cat = listcategory.ListCategory('Foo', before, + columns_to_filter=filter_cols) + cat.set_pattern(pattern) _validate(cat, after) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 2f3b8474d..1b4190d21 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -132,7 +132,7 @@ def test_set_pattern(pattern, filter_cols, before, after): table.insert(row) filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) - cat.set_pattern(pattern, filter_cols) + cat.set_pattern(pattern) _validate(cat, after) From c297f047d224d9d681a45b5712bcb67a80044b55 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 31 May 2017 12:35:43 -0400 Subject: [PATCH 087/337] Don't regenerate completion model needlessly. If the completion model would stay the same, just keep it and update the filter pattern rather than instantiating a new model each time the pattern changes. --- qutebrowser/completion/completer.py | 37 ++++++++++++------- qutebrowser/completion/completionwidget.py | 28 ++++++-------- qutebrowser/completion/models/miscmodels.py | 1 + tests/unit/completion/test_completer.py | 9 ++--- .../unit/completion/test_completionwidget.py | 3 +- 5 files changed, 43 insertions(+), 35 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 94c7f9352..b70325abe 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -39,6 +39,7 @@ class Completer(QObject): _last_cursor_pos: The old cursor position so we avoid double completion updates. _last_text: The old command text so we avoid double completion updates. + _last_completion_func: The completion function used for the last text. """ def __init__(self, cmd, win_id, parent=None): @@ -52,6 +53,7 @@ class Completer(QObject): self._timer.timeout.connect(self._update_completion) self._last_cursor_pos = None self._last_text = None + self._last_completion_func = None self._cmd.update_completion.connect(self.schedule_completion_update) def __repr__(self): @@ -63,7 +65,7 @@ class Completer(QObject): return completion.model() def _get_new_completion(self, before_cursor, under_cursor): - """Get a new completion. + """Get the completion function based on the current command text. Args: before_cursor: The command chunks before the cursor. @@ -81,7 +83,7 @@ class Completer(QObject): if not before_cursor: # '|' or 'set|' log.completion.debug('Starting command completion') - return miscmodels.command() + return miscmodels.command try: cmd = cmdutils.cmd_dict[before_cursor[0]] except KeyError: @@ -90,17 +92,11 @@ class Completer(QObject): return None argpos = len(before_cursor) - 1 try: - completion = cmd.get_pos_arg_info(argpos).completion + func = cmd.get_pos_arg_info(argpos).completion except IndexError: log.completion.debug("No completion in position {}".format(argpos)) return None - if completion is None: - return None - - model = completion(*before_cursor[1:]) - log.completion.debug('Starting {} completion' - .format(completion.__name__)) - return model + return func def _quote(self, s): """Quote s if it needs quoting for the commandline. @@ -223,9 +219,24 @@ class Completer(QObject): before_cursor, pattern, after_cursor)) pattern = pattern.strip("'\"") - model = self._get_new_completion(before_cursor, pattern) - log.completion.debug("Setting pattern to '{}'".format(pattern)) - completion.set_model(model, pattern) + func = self._get_new_completion(before_cursor, pattern) + + if func is None: + log.completion.debug('Clearing completion') + completion.set_model(None) + elif func != self._last_completion_func: + self._last_completion_func = func + args = (x for x in before_cursor[1:] if not x.startswith('-')) + model = func(*args) + log.completion.debug('Starting {} completion' + .format(func.__name__)) + completion.set_model(model) + completion.set_pattern(pattern) + else: + log.completion.debug('Setting pattern {}'.format(pattern)) + completion.set_pattern(pattern) + + self._last_completion_func = None def _change_completed_part(self, newtext, before, after, immediate=False): """Change the part we're currently completing in the commandline. diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 5e207d68d..a476e70dc 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -261,32 +261,29 @@ class CompletionView(QTreeView): elif config.get('completion', 'show') == 'auto': self.show() - def set_model(self, model, pattern=None): + def set_model(self, model): """Switch completion to a new model. Called from on_update_completion(). Args: model: The model to use. - pattern: The filter pattern to set (what the user entered). """ if model is None: self._active = False self.hide() return + if self.model() is not None: + self.model().deleteLater() + self.selectionModel().deleteLater() + model.setParent(self) - old_model = self.model() - if model is not old_model: - sel_model = self.selectionModel() + self.setModel(model) + self._column_widths = model.column_widths + self._active = True - self.setModel(model) - self._active = True - - if sel_model is not None: - sel_model.deleteLater() - if old_model is not None: - old_model.deleteLater() + self.set_pattern('') if (config.get('completion', 'show') == 'always' and model.count() > 0): @@ -297,12 +294,11 @@ class CompletionView(QTreeView): for i in range(model.rowCount()): self.expand(model.index(i, 0)) - if pattern is not None: - model.set_pattern(pattern) - - self._column_widths = model.column_widths + def set_pattern(self, pattern): + self.model().set_pattern(pattern) self._resize_columns() self._maybe_update_geometry() + self.show() def _maybe_update_geometry(self): """Emit the update_geometry signal if the config says so.""" diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 62818f767..233ee8dcc 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -137,6 +137,7 @@ def bind(_key): _key: the key being bound. """ # TODO: offer a 'Current binding' completion based on the key. + print(_key) model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) model.add_category(listcategory.ListCategory("Commands", cmdlist)) diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index c407aeab8..537baad46 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -191,15 +191,14 @@ def test_update_completion(txt, kind, pattern, pos_args, status_command_stub, # this test uses | as a placeholder for the current cursor position _set_cmd_prompt(status_command_stub, txt) completer_obj.schedule_completion_update() - assert completion_widget_stub.set_model.call_count == 1 - args = completion_widget_stub.set_model.call_args[0] if kind is None: - assert args[0] is None + assert completion_widget_stub.set_pattern.call_count == 0 else: - model = args[0] + assert completion_widget_stub.set_model.call_count == 1 + model = completion_widget_stub.set_model.call_args[0][0] assert model.kind == kind assert model.pos_args == pos_args - assert args[1] == pattern + completion_widget_stub.set_pattern.assert_called_once_with(pattern) @pytest.mark.parametrize('before, newtxt, after', [ diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 7985bfa5b..66f6e7f89 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -84,7 +84,8 @@ def test_set_model(completionview): def test_set_pattern(completionview): model = completionmodel.CompletionModel() model.set_pattern = mock.Mock() - completionview.set_model(model, 'foo') + completionview.set_model(model) + completionview.set_pattern('foo') model.set_pattern.assert_called_with('foo') From a01c76db54a8a997a1d46d411aa840e5a6ed7436 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 2 Jun 2017 07:47:28 -0400 Subject: [PATCH 088/337] Remove 'group by' from url completion query. This seemed to have a significant performance impact. Removing it means that instead of just seeing the most recent atime for a given url, you will see multiple entries. --- qutebrowser/completion/models/urlmodel.py | 4 ++-- tests/unit/completion/test_models.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 9e2571848..b106da366 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -75,9 +75,9 @@ def url(): columns_to_filter=[0, 1])) timefmt = config.get('completion', 'timestamp-format') - select_time = "strftime('{}', max(atime), 'unixepoch')".format(timefmt) + select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'History', sort_order='desc', sort_by='atime', group_by='url', + 'History', sort_order='desc', sort_by='atime', filter_fields=['url', 'title'], select='url, title, {}'.format(select_time), where='not redirect') model.add_category(hist_cat) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index c46287926..46730cc21 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -306,7 +306,9 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, "History": [ ('https://github.com', 'https://github.com', '2016-05-01'), ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + ('https://python.org', 'Welcome to Python.org', '2016-02-08'), ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ('https://python.org', 'Welcome to Python.org', '2014-03-08'), ], }) From 42243d3d9765ac52c3cf2185552d34224a5805b2 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 5 Jun 2017 07:40:48 -0400 Subject: [PATCH 089/337] Add more performance logging to completion. --- qutebrowser/completion/completer.py | 11 ++++++----- qutebrowser/completion/completionwidget.py | 11 ++++++----- qutebrowser/completion/models/listcategory.py | 17 ++++++++--------- tests/end2end/features/test_completion_bdd.py | 2 +- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index b70325abe..ae2e9e4bb 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -23,7 +23,7 @@ from PyQt5.QtCore import pyqtSlot, QObject, QTimer from qutebrowser.config import config from qutebrowser.commands import cmdutils, runners -from qutebrowser.utils import log, utils +from qutebrowser.utils import log, utils, debug from qutebrowser.completion.models import miscmodels @@ -227,10 +227,11 @@ class Completer(QObject): elif func != self._last_completion_func: self._last_completion_func = func args = (x for x in before_cursor[1:] if not x.startswith('-')) - model = func(*args) - log.completion.debug('Starting {} completion' - .format(func.__name__)) - completion.set_model(model) + with debug.log_time(log.completion, + 'Instantiate {} completion'.format(func.__name__)): + model = func(*args) + with debug.log_time(log.completion, 'Set completion model'): + completion.set_model(model) completion.set_pattern(pattern) else: log.completion.debug('Setting pattern {}'.format(pattern)) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index a476e70dc..1f8b3d9f4 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -28,7 +28,7 @@ from PyQt5.QtCore import pyqtSlot, pyqtSignal, Qt, QItemSelectionModel, QSize from qutebrowser.config import config, style from qutebrowser.completion import completiondelegate -from qutebrowser.utils import utils, usertypes, objreg +from qutebrowser.utils import utils, usertypes, objreg, debug, log from qutebrowser.commands import cmdexc, cmdutils @@ -295,10 +295,11 @@ class CompletionView(QTreeView): self.expand(model.index(i, 0)) def set_pattern(self, pattern): - self.model().set_pattern(pattern) - self._resize_columns() - self._maybe_update_geometry() - self.show() + with debug.log_time(log.completion, 'Set pattern {}'.format(pattern)): + self.model().set_pattern(pattern) + self._resize_columns() + self._maybe_update_geometry() + self.show() def _maybe_update_geometry(self): """Emit the update_geometry signal if the config says so.""" diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index fbad3b063..75cc2642c 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -52,15 +52,14 @@ class ListCategory(QSortFilterProxyModel): Args: val: The value to set. """ - with debug.log_time(log.completion, 'Setting filter pattern'): - self.pattern = val - val = re.sub(r' +', r' ', val) # See #1919 - val = re.escape(val) - val = val.replace(r'\ ', '.*') - self.pattern_re = re.compile(val, re.IGNORECASE) - self.invalidate() - sortcol = 0 - self.sort(sortcol) + self.pattern = val + val = re.sub(r' +', r' ', val) # See #1919 + val = re.escape(val) + val = val.replace(r'\ ', '.*') + self.pattern_re = re.compile(val, re.IGNORECASE) + self.invalidate() + sortcol = 0 + self.sort(sortcol) def filterAcceptsRow(self, row, parent): """Custom filter implementation. diff --git a/tests/end2end/features/test_completion_bdd.py b/tests/end2end/features/test_completion_bdd.py index 4a06bfbc0..82e2df030 100644 --- a/tests/end2end/features/test_completion_bdd.py +++ b/tests/end2end/features/test_completion_bdd.py @@ -24,5 +24,5 @@ bdd.scenarios('completion.feature') @bdd.then(bdd.parsers.parse("the completion model should be {model}")) def check_model(quteproc, model): """Make sure the completion model was set to something.""" - pattern = "Starting {} completion".format(model) + pattern = "Starting {} completion *".format(model) quteproc.wait_for(message=pattern) From 565ba23f8cd33fd777324a85794cd3dc7dc4b72f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 6 Jun 2017 20:40:24 -0400 Subject: [PATCH 090/337] Don't instantiate completion models nedlessly. For real this time. A mistake on the last commit like this meant models were still spuriously instantiated. Now that the completion model is reused, the layoutChanged signal needs to be forwarded through, otherwise the view will not update. --- qutebrowser/completion/completer.py | 13 ++++++------- qutebrowser/completion/completionwidget.py | 2 -- qutebrowser/completion/models/completionmodel.py | 1 + tests/unit/completion/test_completionmodel.py | 4 ++-- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index ae2e9e4bb..491fe2e04 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -224,20 +224,19 @@ class Completer(QObject): if func is None: log.completion.debug('Clearing completion') completion.set_model(None) - elif func != self._last_completion_func: + self._last_completion_func = None + return + + if func != self._last_completion_func: self._last_completion_func = func args = (x for x in before_cursor[1:] if not x.startswith('-')) with debug.log_time(log.completion, - 'Instantiate {} completion'.format(func.__name__)): + 'Starting {} completion'.format(func.__name__)): model = func(*args) with debug.log_time(log.completion, 'Set completion model'): completion.set_model(model) - completion.set_pattern(pattern) - else: - log.completion.debug('Setting pattern {}'.format(pattern)) - completion.set_pattern(pattern) - self._last_completion_func = None + completion.set_pattern(pattern) def _change_completed_part(self, newtext, before, after, immediate=False): """Change the part we're currently completing in the commandline. diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 1f8b3d9f4..711301b4c 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -283,8 +283,6 @@ class CompletionView(QTreeView): self._column_widths = model.column_widths self._active = True - self.set_pattern('') - if (config.get('completion', 'show') == 'always' and model.count() > 0): self.show() diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 9543792ed..cbc20470c 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -63,6 +63,7 @@ class CompletionModel(QAbstractItemModel): def add_category(self, cat): """Add a completion category to the model.""" self._categories.append(cat) + cat.layoutChanged.connect(self.layoutChanged) def data(self, index, role=Qt.DisplayRole): """Return the item data for index. diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 8d43f2883..8f8acced2 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -56,7 +56,7 @@ def test_first_last_item(counts): def test_count(counts): model = completionmodel.CompletionModel() for c in counts: - cat = mock.Mock(spec=['rowCount']) + cat = mock.Mock(spec=['rowCount', 'layoutChanged']) cat.rowCount = mock.Mock(return_value=c) model.add_category(cat) assert model.count() == sum(counts) @@ -66,7 +66,7 @@ def test_count(counts): def test_set_pattern(pat): """Validate the filtering and sorting results of set_pattern.""" model = completionmodel.CompletionModel() - cats = [mock.Mock(spec=['set_pattern'])] * 3 + cats = [mock.Mock(spec=['set_pattern', 'layoutChanged'])] * 3 for c in cats: c.set_pattern = mock.Mock() model.add_category(c) From 6a0fc5afd26c10aff4239d25ae1199fb32e6419b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 6 Jun 2017 21:35:57 -0400 Subject: [PATCH 091/337] Create a SQL index on History.url. This will hopefully speed up historyContains but does not seem to speed up the completion query, unfortunately. --- qutebrowser/browser/history.py | 1 + qutebrowser/misc/sql.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index d8b1f81d5..5abe8d9d6 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -78,6 +78,7 @@ class WebHistory(sql.SqlTable): def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) + self.create_index('HistoryIndex', 'url', where='not redirect') self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 0b81fc0f9..b7e180a24 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -122,6 +122,18 @@ class SqlTable(QObject): # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) + def create_index(self, name, field, where): + """Create an index over this table. + + Args: + name: Name of the index, should be unique. + field: Name of the field to index. + where: WHERE clause for a partial index. + """ + q = Query("CREATE INDEX IF NOT EXISTS {} ON {} ({}) WHERE {}" + .format(name, self._name, field, where)) + q.run() + def __iter__(self): """Iterate rows in the table.""" q = Query("SELECT * FROM {}".format(self._name)) From 5b827cf86ad2d3e1f307fd4920f9768eeaf70745 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 6 Jun 2017 21:40:17 -0400 Subject: [PATCH 092/337] Fix typo in sql exception handling --- qutebrowser/misc/sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index b7e180a24..416229ae8 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -81,8 +81,8 @@ class Query(QSqlQuery): self.addBindValue(val) log.sql.debug('self bindings: {}'.format(self.boundValues())) if not self.exec_(): - raise SqlException('Failed to exec self "{}": "{}"'.format( - self.lastself(), self.lastError().text())) + raise SqlException('Failed to exec query "{}": "{}"'.format( + self.lastQuery(), self.lastError().text())) def value(self): """Return the result of a single-value query (e.g. an EXISTS).""" From 478a719f77f99171b790020a773a8a5eee1efb5f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 6 Jun 2017 21:48:58 -0400 Subject: [PATCH 093/337] Use a prepared query for historyContains. This is called often, hopefully a prepared query will speed it up. This also modifies Query.run to return self for easier chaining, so you can use `query.run.value()` instead of `query.run` ; query.value()`. --- qutebrowser/browser/history.py | 3 ++- qutebrowser/misc/sql.py | 12 +++++------- tests/unit/misc/test_sql.py | 23 ++++++++++++++--------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 5abe8d9d6..008773e62 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -79,6 +79,7 @@ class WebHistory(sql.SqlTable): super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) self.create_index('HistoryIndex', 'url', where='not redirect') + self._contains_query = self.contains_query('url') self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' @@ -97,7 +98,7 @@ class WebHistory(sql.SqlTable): return utils.get_repr(self, length=len(self)) def __contains__(self, url): - return self.contains('url', url) + return self._contains_query.run([url]).value() def _add_entry(self, entry): """Add an entry to the in-memory database.""" diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 416229ae8..b63979c0c 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -83,6 +83,7 @@ class Query(QSqlQuery): if not self.exec_(): raise SqlException('Failed to exec query "{}": "{}"'.format( self.lastQuery(), self.lastError().text())) + return self def value(self): """Return the result of a single-value query (e.g. an EXISTS).""" @@ -140,17 +141,14 @@ class SqlTable(QObject): q.run() return iter(q) - def contains(self, field, value): - """Return whether the table contains the matching item. + def contains_query(self, field): + """Return a prepared query that checks for the existence of an item. Args: field: Field to match. - value: Value to check for the given field. """ - q = Query("Select EXISTS(SELECT * FROM {} where {} = ?)" - .format(self._name, field)) - q.run([value]) - return q.value() + return Query("Select EXISTS(SELECT * FROM {} where {} = ?)" + .format(self._name, field)) def __len__(self): """Return the count of rows in the table.""" diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index ca736b0ad..03cb0f27c 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -94,15 +94,20 @@ def test_contains(): table.insert(['one', 1, False]) table.insert(['nine', 9, False]) table.insert(['thirteen', 13, True]) - assert table.contains('name', 'one') - assert table.contains('name', 'thirteen') - assert table.contains('val', 9) - assert table.contains('lucky', False) - assert table.contains('lucky', True) - assert not table.contains('name', 'oone') - assert not table.contains('name', 1) - assert not table.contains('name', '*') - assert not table.contains('val', 10) + + name_query = table.contains_query('name') + val_query = table.contains_query('val') + lucky_query = table.contains_query('lucky') + + assert name_query.run(['one']).value() + assert name_query.run(['thirteen']).value() + assert val_query.run([9]).value() + assert lucky_query.run([False]).value() + assert lucky_query.run([True]).value() + assert not name_query.run(['oone']).value() + assert not name_query.run([1]).value() + assert not name_query.run(['*']).value() + assert not val_query.run([10]).value() def test_delete_all(qtbot): From ea0b3eee053a41c489bcf72f43a72e5ca87b6278 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 07:33:36 -0400 Subject: [PATCH 094/337] Use full, not partial index for history. historyContains includes redirect urls, so we actually don't want a partial index here. --- qutebrowser/browser/history.py | 2 +- qutebrowser/misc/sql.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 008773e62..c64674fdf 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -78,7 +78,7 @@ class WebHistory(sql.SqlTable): def __init__(self, parent=None): super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) - self.create_index('HistoryIndex', 'url', where='not redirect') + self.create_index('HistoryIndex', 'url') self._contains_query = self.contains_query('url') self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index b63979c0c..0435d2ea3 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -123,16 +123,15 @@ class SqlTable(QObject): # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) - def create_index(self, name, field, where): + def create_index(self, name, field): """Create an index over this table. Args: name: Name of the index, should be unique. field: Name of the field to index. - where: WHERE clause for a partial index. """ - q = Query("CREATE INDEX IF NOT EXISTS {} ON {} ({}) WHERE {}" - .format(name, self._name, field, where)) + q = Query("CREATE INDEX IF NOT EXISTS {} ON {} ({})" + .format(name, self._name, field)) q.run() def __iter__(self): From 309b6ba32c5918a728d6c49511fa7553902e0d0f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 08:10:44 -0400 Subject: [PATCH 095/337] Move _import_history to history.py. Also adjusts the history import test to operate at a higher level and ensure the old text file is removed (or isn't, in the case of an error). --- qutebrowser/app.py | 23 +------------------ qutebrowser/browser/history.py | 28 ++++++++++++++++++++--- tests/helpers/stubs.py | 4 ++++ tests/unit/browser/webkit/test_history.py | 18 ++++++++++----- 4 files changed, 42 insertions(+), 31 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 0f95c1c5d..ecfd456be 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -161,7 +161,7 @@ def init(args, crash_handler): QDesktopServices.setUrlHandler('https', open_desktopservices_url) QDesktopServices.setUrlHandler('qute', open_desktopservices_url) - _import_history() + objreg.get('history').import_txt() log.init.debug("Init done!") crash_handler.raise_crashdlg() @@ -479,27 +479,6 @@ def _init_modules(args, crash_handler): browsertab.init() -def _import_history(): - """Import a history text file into sqlite if it exists. - - In older versions of qutebrowser, history was stored in a text format. - This converts that file into the new sqlite format and removes it. - """ - path = os.path.join(standarddir.data(), 'history') - if not os.path.isfile(path): - return - - def action(): - with debug.log_time(log.init, 'Converting old history file to sqlite'): - objreg.get('web-history').read(path) - message.info('History import complete. Removing {}'.format(path)) - os.remove(path) - - # delay to give message time to appear before locking down for import - message.info('Converting {} to sqlite...'.format(path)) - QTimer.singleShot(100, action) - - class Quitter: """Utility class to quit/restart the QApplication. diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index c64674fdf..cb10c2bb2 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -22,10 +22,11 @@ import os import time -from PyQt5.QtCore import pyqtSlot, QUrl +from PyQt5.QtCore import pyqtSlot, QUrl, QTimer from qutebrowser.commands import cmdutils -from qutebrowser.utils import utils, objreg, log, qtutils, usertypes, message +from qutebrowser.utils import (utils, objreg, log, qtutils, usertypes, message, + debug, standarddir) from qutebrowser.misc import objects, sql @@ -217,7 +218,28 @@ class WebHistory(sql.SqlTable): return (url, title, float(atime), bool(redirect)) - def read(self, path): + def import_txt(self): + """Import a history text file into sqlite if it exists. + + In older versions of qutebrowser, history was stored in a text format. + This converts that file into the new sqlite format and removes it. + """ + path = os.path.join(standarddir.data(), 'history') + if not os.path.isfile(path): + return + + def action(): + with debug.log_time(log.init, 'Import old history file to sqlite'): + self._read(path) + message.info('History import complete. Removing {}' + .format(path)) + os.remove(path) + + # delay to give message time to appear before locking down for import + message.info('Converting {} to sqlite...'.format(path)) + QTimer.singleShot(100, action) + + def _read(self, path): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: rows = [] diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index b852426d2..717ce4c18 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -385,6 +385,10 @@ class InstaTimer(QObject): def setInterval(self, interval): pass + @staticmethod + def singleShot(_interval, fun): + fun() + class FakeConfigType: diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 16b86ca4a..5386dddd9 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -230,8 +230,9 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): assert default_interface is None -def test_read(hist, tmpdir): - histfile = tmpdir / 'history' +def test_read(hist, data_tmpdir, monkeypatch, stubs): + monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) + histfile = data_tmpdir / 'history' # empty line is deliberate, to test skipping empty lines histfile.write('''12345 http://example.com/ title 12346 http://qutebrowser.org/ @@ -239,7 +240,7 @@ def test_read(hist, tmpdir): 68891-r http://example.com/path/other ''') - hist.read(str(histfile)) + hist.import_txt() assert list(hist) == [ ('http://example.com/', 'title', 12345, False), @@ -248,6 +249,8 @@ def test_read(hist, tmpdir): ('http://example.com/path/other', '', 68891, True) ] + assert not histfile.exists() + @pytest.mark.parametrize('line', [ 'xyz http://example.com/bad-timestamp', @@ -255,12 +258,15 @@ def test_read(hist, tmpdir): 'http://example.com/no-timestamp', '68891-r-r http://example.com/double-flag', ]) -def test_read_invalid(hist, tmpdir, line): - histfile = tmpdir / 'history' +def test_read_invalid(hist, data_tmpdir, line, monkeypatch, stubs): + monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) + histfile = data_tmpdir / 'history' histfile.write(line) with pytest.raises(Exception): - hist.read(str(histfile)) + hist.import_txt() + + assert histfile.exists() def test_debug_dump_history(hist, tmpdir): From f8325cbbc1c8cade9c47ca922228fb150032af69 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 08:15:10 -0400 Subject: [PATCH 096/337] Remove print statement --- qutebrowser/completion/models/miscmodels.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 233ee8dcc..62818f767 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -137,7 +137,6 @@ def bind(_key): _key: the key being bound. """ # TODO: offer a 'Current binding' completion based on the key. - print(_key) model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) model.add_category(listcategory.ListCategory("Commands", cmdlist)) From 389e1b017895e67e93e697ffd36b0114dbe122ba Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 08:40:13 -0400 Subject: [PATCH 097/337] Fix bad objreg reference in app.py. --- qutebrowser/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index ecfd456be..8f7984f2d 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -161,7 +161,7 @@ def init(args, crash_handler): QDesktopServices.setUrlHandler('https', open_desktopservices_url) QDesktopServices.setUrlHandler('qute', open_desktopservices_url) - objreg.get('history').import_txt() + objreg.get('web-history').import_txt() log.init.debug("Init done!") crash_handler.raise_crashdlg() From 7f27603772295a22f91b5f566d1b044f5a1a0621 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 09:10:05 -0400 Subject: [PATCH 098/337] Fix columns_to_filter for sql category. --- qutebrowser/completion/models/sqlcategory.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index b8cf07ffa..c3b5fd6c2 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -61,8 +61,11 @@ class SqlCategory(QSqlQueryModel): self._query = sql.Query(querystr) self._param_count = len(filter_fields) - rec = self._query.record() - # will this work? + self.columns_to_filter = None + + # map filter_fields to indices + col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name)) + rec = col_query.run().record() self.columns_to_filter = [rec.indexOf(n) for n in filter_fields] def set_pattern(self, pattern): @@ -72,8 +75,6 @@ class SqlCategory(QSqlQueryModel): pattern: string pattern to filter by. columns_to_filter: indices of columns to apply pattern to. """ - # TODO: eliminate columns_to_filter - #assert len(columns_to_filter) == self._param_count # escape to treat a user input % or _ as a literal, not a wildcard pattern = pattern.replace('%', '\\%') pattern = pattern.replace('_', '\\_') From c64b7d00e6a793f9d34af1ac294e53a626d7fe98 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 7 Jun 2017 15:57:56 +0200 Subject: [PATCH 099/337] Add separate table for history visits --- qutebrowser/browser/history.py | 29 ++++++++++++++++---- qutebrowser/completion/models/sqlcategory.py | 2 +- qutebrowser/completion/models/urlmodel.py | 4 +-- qutebrowser/misc/sql.py | 29 ++++++++++++++++---- 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index cb10c2bb2..718ad132d 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -72,15 +72,28 @@ class Entry: return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) +class HistoryVisits(sql.SqlTable): + + """Secondary table with visited URLs and timestamps.""" + + def __init__(self, parent=None): + super().__init__("Visits", ['url', 'atime'], + fkeys={'url': 'History(url)'}) + + class WebHistory(sql.SqlTable): """The global history of visited pages.""" def __init__(self, parent=None): - super().__init__("History", ['url', 'title', 'atime', 'redirect'], + super().__init__("History", + ['url', 'title', 'last_atime', 'redirect'], + constraints={'url': 'PRIMARY KEY'}, parent=parent) + self.visits = HistoryVisits(parent=self) self.create_index('HistoryIndex', 'url') self._contains_query = self.contains_query('url') + # FIXME self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' @@ -104,7 +117,8 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, int(entry.atime), - entry.redirect]) + entry.redirect], replace=True) + self.visits.insert([entry.url_str(), int(entry.atime)]) def get_recent(self): """Get the most recent history entries.""" @@ -216,7 +230,8 @@ class WebHistory(sql.SqlTable): redirect = 'r' in flags - return (url, title, float(atime), bool(redirect)) + return ((url, float(atime)), + (url, title, float(atime), bool(redirect))) def import_txt(self): """Import a history text file into sqlite if it exists. @@ -243,17 +258,20 @@ class WebHistory(sql.SqlTable): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: rows = [] + visit_rows = [] for (i, line) in enumerate(f): line = line.strip() if not line: continue try: - row = self._parse_entry(line.strip()) + visit_row, row = self._parse_entry(line.strip()) rows.append(row) + visit_rows.append(visit_row) except ValueError: raise Exception('Failed to parse line #{} of {}: "{}"' .format(i, path, line)) - self.insert_batch(rows) + self.insert_batch(rows, replace=True) + self.visits.insert_batch(visit_rows) @cmdutils.register(instance='web-history', debug=True) def debug_dump_history(self, dest): @@ -262,6 +280,7 @@ class WebHistory(sql.SqlTable): Args: dest: Where to write the file to. """ + # FIXME dest = os.path.expanduser(dest) lines = ('{}{} {} {}' diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index c3b5fd6c2..d363d09cb 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -30,7 +30,7 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, - select='*', where=None, group_by=None, parent=None): + select='*', where=None, group_by=None, suffix=None, parent=None): """Create a new completion category backed by a sql table. Args: diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index b106da366..1be364aba 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -75,9 +75,9 @@ def url(): columns_to_filter=[0, 1])) timefmt = config.get('completion', 'timestamp-format') - select_time = "strftime('{}', atime, 'unixepoch')".format(timefmt) + select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'History', sort_order='desc', sort_by='atime', + 'History', sort_order='desc', sort_by='last_atime', filter_fields=['url', 'title'], select='url, title, {}'.format(select_time), where='not redirect') model.add_category(hist_cat) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 0435d2ea3..dec051959 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -106,7 +106,8 @@ class SqlTable(QObject): changed = pyqtSignal() - def __init__(self, name, fields, parent=None): + def __init__(self, name, fields, constraints=None, fkeys=None, + parent=None): """Create a new table in the sql database. Raises SqlException if the table already exists. @@ -114,11 +115,23 @@ class SqlTable(QObject): Args: name: Name of the table. fields: A list of field names. + constraints: A dict mapping field names to constraint strings. + fkeys: A dict mapping field names to foreign keys. """ super().__init__(parent) self._name = name + + constraints = constraints or {} + fkeys = fkeys or {} + + column_defs = ['{} {}'.format(field, constraints.get(field, '')) + for field in fields] + for field, fkey in sorted(fkeys.items()): + column_defs.append('FOREIGN KEY({}) REFERENCES {}'.format( + field, fkey)) + q = Query("CREATE TABLE IF NOT EXISTS {} ({})" - .format(name, ','.join(fields))) + .format(name, ','.join(column_defs))) q.run() # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) @@ -171,25 +184,29 @@ class SqlTable(QObject): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, values): + def insert(self, values, replace=False): """Append a row to the table. Args: values: A list of values to insert. + replace: If set, replace existing values. """ paramstr = ','.join(['?'] * len(values)) - q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q = Query("INSERT {} INTO {} values({})".format( + 'OR REPLACE' if replace else '', self._name, paramstr)) q.run(values) self.changed.emit() - def insert_batch(self, rows): + def insert_batch(self, rows, replace=False): """Performantly append multiple rows to the table. Args: rows: A list of lists, where each sub-list is a row. + replace: If set, replace existing values. """ paramstr = ','.join(['?'] * len(rows[0])) - q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q = Query("INSERT {} INTO {} values({})".format( + 'OR REPLACE' if replace else '', self._name, paramstr)) transposed = [list(row) for row in zip(*rows)] for val in transposed: From 57d96a45120e581668e7ac8dfeabd59baee8de8c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 7 Jun 2017 16:02:10 +0200 Subject: [PATCH 100/337] Add a CompletionHistory instead of HistoryVisits table --- qutebrowser/browser/history.py | 38 ++++++++++---------- qutebrowser/completion/models/sqlcategory.py | 2 +- qutebrowser/completion/models/urlmodel.py | 4 +-- qutebrowser/misc/sql.py | 11 ++---- tests/unit/completion/test_models.py | 21 +++++------ 5 files changed, 33 insertions(+), 43 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 718ad132d..5e950f111 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -72,13 +72,13 @@ class Entry: return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) -class HistoryVisits(sql.SqlTable): +class CompletionHistory(sql.SqlTable): - """Secondary table with visited URLs and timestamps.""" + """History which only has the newest entry for each URL.""" def __init__(self, parent=None): - super().__init__("Visits", ['url', 'atime'], - fkeys={'url': 'History(url)'}) + super().__init__("CompletionHistory", ['url', 'title', 'last_atime'], + constraints={'url': 'PRIMARY KEY'}, parent=parent) class WebHistory(sql.SqlTable): @@ -86,14 +86,11 @@ class WebHistory(sql.SqlTable): """The global history of visited pages.""" def __init__(self, parent=None): - super().__init__("History", - ['url', 'title', 'last_atime', 'redirect'], - constraints={'url': 'PRIMARY KEY'}, + super().__init__("History", ['url', 'title', 'atime', 'redirect'], parent=parent) - self.visits = HistoryVisits(parent=self) + self.completion = CompletionHistory(parent=self) self.create_index('HistoryIndex', 'url') self._contains_query = self.contains_query('url') - # FIXME self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' @@ -117,8 +114,10 @@ class WebHistory(sql.SqlTable): def _add_entry(self, entry): """Add an entry to the in-memory database.""" self.insert([entry.url_str(), entry.title, int(entry.atime), - entry.redirect], replace=True) - self.visits.insert([entry.url_str(), int(entry.atime)]) + entry.redirect]) + if not entry.redirect: + self.completion.insert([entry.url_str(), entry.title, + int(entry.atime)], replace=True) def get_recent(self): """Get the most recent history entries.""" @@ -229,9 +228,10 @@ class WebHistory(sql.SqlTable): raise ValueError("Invalid flags {!r}".format(flags)) redirect = 'r' in flags + row = (url, title, float(atime), redirect) + completion_row = None if redirect else (url, title, float(atime)) - return ((url, float(atime)), - (url, title, float(atime), bool(redirect))) + return (row, completion_row) def import_txt(self): """Import a history text file into sqlite if it exists. @@ -258,20 +258,21 @@ class WebHistory(sql.SqlTable): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: rows = [] - visit_rows = [] + completion_rows = [] for (i, line) in enumerate(f): line = line.strip() if not line: continue try: - visit_row, row = self._parse_entry(line.strip()) + row, completion_row = self._parse_entry(line.strip()) rows.append(row) - visit_rows.append(visit_row) + if completion_row is not None: + completion_rows.append(completion_row) except ValueError: raise Exception('Failed to parse line #{} of {}: "{}"' .format(i, path, line)) - self.insert_batch(rows, replace=True) - self.visits.insert_batch(visit_rows) + self.insert_batch(rows) + self.completion.insert_batch(completion_rows, replace=True) @cmdutils.register(instance='web-history', debug=True) def debug_dump_history(self, dest): @@ -280,7 +281,6 @@ class WebHistory(sql.SqlTable): Args: dest: Where to write the file to. """ - # FIXME dest = os.path.expanduser(dest) lines = ('{}{} {} {}' diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index d363d09cb..c3b5fd6c2 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -30,7 +30,7 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, - select='*', where=None, group_by=None, suffix=None, parent=None): + select='*', where=None, group_by=None, parent=None): """Create a new completion category backed by a sql table. Args: diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 1be364aba..2c92815c9 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -77,8 +77,8 @@ def url(): timefmt = config.get('completion', 'timestamp-format') select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'History', sort_order='desc', sort_by='last_atime', + 'CompletionHistory', sort_order='desc', sort_by='last_atime', filter_fields=['url', 'title'], - select='url, title, {}'.format(select_time), where='not redirect') + select='url, title, {}'.format(select_time)) model.add_category(hist_cat) return model diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index dec051959..f174248b3 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -106,8 +106,7 @@ class SqlTable(QObject): changed = pyqtSignal() - def __init__(self, name, fields, constraints=None, fkeys=None, - parent=None): + def __init__(self, name, fields, constraints=None, parent=None): """Create a new table in the sql database. Raises SqlException if the table already exists. @@ -116,22 +115,16 @@ class SqlTable(QObject): name: Name of the table. fields: A list of field names. constraints: A dict mapping field names to constraint strings. - fkeys: A dict mapping field names to foreign keys. """ super().__init__(parent) self._name = name constraints = constraints or {} - fkeys = fkeys or {} - column_defs = ['{} {}'.format(field, constraints.get(field, '')) for field in fields] - for field, fkey in sorted(fkeys.items()): - column_defs.append('FOREIGN KEY({}) REFERENCES {}'.format( - field, fkey)) - q = Query("CREATE TABLE IF NOT EXISTS {} ({})" .format(name, ','.join(column_defs))) + q.run() # pylint: disable=invalid-name self.Entry = collections.namedtuple(name + '_Entry', fields) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 46730cc21..c85ecbf38 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -155,24 +155,22 @@ def bookmarks(bookmark_manager_stub): @pytest.fixture def web_history_stub(stubs, init_sql): - return sql.SqlTable("History", ['url', 'title', 'atime', 'redirect']) + return sql.SqlTable("CompletionHistory", ['url', 'title', 'last_atime']) @pytest.fixture def web_history(web_history_stub, init_sql): """Pre-populate the web-history database.""" - web_history_stub.insert(['http://some-redirect.example.com', 'redirect', - datetime(2016, 9, 5).timestamp(), True]) web_history_stub.insert(['http://qutebrowser.org', 'qutebrowser', - datetime(2015, 9, 5).timestamp(), False]) + datetime(2015, 9, 5).timestamp()]) web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2016, 2, 8).timestamp(), False]) + datetime(2016, 2, 8).timestamp()]) web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2016, 3, 8).timestamp(), False]) + datetime(2016, 3, 8).timestamp()]) web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2014, 3, 8).timestamp(), False]) + datetime(2014, 3, 8).timestamp()]) web_history_stub.insert(['https://github.com', 'https://github.com', - datetime(2016, 5, 1).timestamp(), False]) + datetime(2016, 5, 1).timestamp()]) return web_history_stub @@ -336,7 +334,7 @@ def test_url_completion_pattern(config_stub, web_history_stub, url, title, pattern, rowcount): """Test that url completion filters by url and title.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} - web_history_stub.insert([url, title, 0, False]) + web_history_stub.insert([url, title, 0]) model = urlmodel.url() model.set_pattern(pattern) # 2, 0 is History @@ -582,10 +580,9 @@ def test_url_completion_benchmark(benchmark, config_stub, 'web-history-max-items': 1000} entries = [web_history_stub.Entry( - atime=i, + last_atime=i, url='http://example.com/{}'.format(i), - title='title{}'.format(i), - redirect=False) + title='title{}'.format(i)) for i in range(100000)] web_history_stub.insert_batch(entries) From 6ce52f39ae28be0aec4243c236b2a03b6a0d5b9d Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 7 Jun 2017 16:37:52 +0200 Subject: [PATCH 101/337] Add debug timings for SQL --- qutebrowser/browser/webkit/webkithistory.py | 5 ++++- qutebrowser/completion/models/sqlcategory.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/webkit/webkithistory.py b/qutebrowser/browser/webkit/webkithistory.py index 453a11883..64b7bf295 100644 --- a/qutebrowser/browser/webkit/webkithistory.py +++ b/qutebrowser/browser/webkit/webkithistory.py @@ -22,6 +22,8 @@ from PyQt5.QtWebKit import QWebHistoryInterface +from qutebrowser.utils import debug + class WebHistoryInterface(QWebHistoryInterface): @@ -48,7 +50,8 @@ class WebHistoryInterface(QWebHistoryInterface): Return: True if the url is in the history, False otherwise. """ - return url_string in self._history + with debug.log_time('sql', 'historyContains'): + return url_string in self._history def init(history): diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index c3b5fd6c2..e2801b072 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -24,6 +24,7 @@ import re from PyQt5.QtSql import QSqlQueryModel from qutebrowser.misc import sql +from qutebrowser.utils import debug class SqlCategory(QSqlQueryModel): @@ -81,5 +82,6 @@ class SqlCategory(QSqlQueryModel): # treat spaces as wildcards to match any of the typed words pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) - self._query.run([pattern] * self._param_count) + with debug.log_time('sql', 'Running completion query'): + self._query.run([pattern] * self._param_count) self.setQuery(self._query) From feed9c8936aaf086bf54eb2ce41a1a260d6cc973 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 20:49:11 -0400 Subject: [PATCH 102/337] Better exception handling in history. - Show an error message when import fails, not a generic crash dialog - Raise CommandError when debug-dump-history fails - Check that the path exists for debug-dump-history --- qutebrowser/browser/history.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 5e950f111..62c773866 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -24,7 +24,7 @@ import time from PyQt5.QtCore import pyqtSlot, QUrl, QTimer -from qutebrowser.commands import cmdutils +from qutebrowser.commands import cmdutils, cmdexc from qutebrowser.utils import (utils, objreg, log, qtutils, usertypes, message, debug, standarddir) from qutebrowser.misc import objects, sql @@ -245,10 +245,14 @@ class WebHistory(sql.SqlTable): def action(): with debug.log_time(log.init, 'Import old history file to sqlite'): - self._read(path) - message.info('History import complete. Removing {}' - .format(path)) - os.remove(path) + try: + self._read(path) + except ValueError as ex: + message.error('Failed to import history: {}'.format(ex)) + else: + message.info('History import complete. Removing {}' + .format(path)) + os.remove(path) # delay to give message time to appear before locking down for import message.info('Converting {} to sqlite...'.format(path)) @@ -268,9 +272,9 @@ class WebHistory(sql.SqlTable): rows.append(row) if completion_row is not None: completion_rows.append(completion_row) - except ValueError: - raise Exception('Failed to parse line #{} of {}: "{}"' - .format(i, path, line)) + except ValueError as ex: + raise ValueError('Failed to parse line #{} of {}: "{}"' + .format(i, path, ex)) self.insert_batch(rows) self.completion.insert_batch(completion_rows, replace=True) @@ -283,6 +287,10 @@ class WebHistory(sql.SqlTable): """ dest = os.path.expanduser(dest) + dirname = os.path.dirname(dest) + if not os.path.exists(dirname): + raise cmdexc.CommandError('Path does not exist', dirname) + lines = ('{}{} {} {}' .format(int(x.atime), '-r' * x.redirect, x.url, x.title) for x in self.select(sort_by='atime', sort_order='asc')) @@ -291,9 +299,8 @@ class WebHistory(sql.SqlTable): try: f.write('\n'.join(lines)) except OSError as e: - message.error('Could not write history: {}'.format(e)) - else: - message.info("Dumped history to {}.".format(dest)) + raise cmdexc.CommandError('Could not write history: {}', e) + message.info("Dumped history to {}.".format(dest)) def init(parent=None): From 6fc61d12fc538c4e129b1e3f31f2cfc8dee6bfdf Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 7 Jun 2017 22:45:14 -0400 Subject: [PATCH 103/337] Assorted small fixes for sql code review. --- qutebrowser/browser/qutescheme.py | 2 +- qutebrowser/completion/completionwidget.py | 19 +++++++------ .../completion/models/completionmodel.py | 12 ++++---- qutebrowser/completion/models/listcategory.py | 3 +- qutebrowser/completion/models/sqlcategory.py | 6 ++-- qutebrowser/completion/models/urlmodel.py | 3 +- qutebrowser/misc/sql.py | 28 +++++++++++++------ tests/end2end/features/misc.feature | 2 -- tests/end2end/features/test_history_bdd.py | 19 ++++++------- tests/helpers/utils.py | 13 +++++++++ tests/unit/browser/test_qutescheme.py | 22 --------------- tests/unit/browser/webkit/test_history.py | 20 ++++--------- .../unit/completion/test_completionwidget.py | 4 +-- tests/unit/completion/test_listcategory.py | 16 ++--------- tests/unit/completion/test_sqlcategory.py | 24 ++++------------ tests/unit/utils/test_version.py | 2 +- 16 files changed, 83 insertions(+), 112 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 4ebc626d3..0c495d28b 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -213,7 +213,7 @@ def qute_history(url): offset = QUrlQuery(url).queryItemValue("offset") offset = int(offset) if offset else None except ValueError as e: - raise QuteSchemeError("Query parameter start_time is invalid", e) + raise QuteSchemeError("Query parameter offset is invalid", e) # Use start_time in query or current time. try: start_time = QUrlQuery(url).queryItemValue("start_time") diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 711301b4c..f4d4d80b7 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -111,7 +111,6 @@ class CompletionView(QTreeView): # objreg.get('config').changed.connect(self.init_command_completion) objreg.get('config').changed.connect(self._on_config_changed) - self._column_widths = (30, 70, 0) self._active = False self._delegate = completiondelegate.CompletionItemDelegate(self) @@ -150,7 +149,8 @@ class CompletionView(QTreeView): def _resize_columns(self): """Resize the completion columns based on column_widths.""" width = self.size().width() - pixel_widths = [(width * perc // 100) for perc in self._column_widths] + column_widths = self.model.column_widths + pixel_widths = [(width * perc // 100) for perc in column_widths] if self.verticalScrollBar().isVisible(): delta = self.style().pixelMetric(QStyle.PM_ScrollBarExtent) + 5 @@ -280,14 +280,8 @@ class CompletionView(QTreeView): model.setParent(self) self.setModel(model) - self._column_widths = model.column_widths self._active = True - - if (config.get('completion', 'show') == 'always' and - model.count() > 0): - self.show() - else: - self.hide() + self._maybe_show() for i in range(model.rowCount()): self.expand(model.index(i, 0)) @@ -297,7 +291,14 @@ class CompletionView(QTreeView): self.model().set_pattern(pattern) self._resize_columns() self._maybe_update_geometry() + self._maybe_show() + + def _maybe_show(self): + if (config.get('completion', 'show') == 'always' and + model.count() > 0): self.show() + else: + self.hide() def _maybe_update_geometry(self): """Emit the update_geometry signal if the config says so.""" diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index cbc20470c..2dec0ed46 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -78,12 +78,14 @@ class CompletionModel(QAbstractItemModel): if not index.isValid() or role != Qt.DisplayRole: return None if not index.parent().isValid(): + # category header if index.column() == 0: return self._categories[index.row()].name - else: - cat = self._categories[index.parent().row()] - idx = cat.index(index.row(), index.column()) - return cat.data(idx) + return None + # item + cat = self._categories[index.parent().row()] + idx = cat.index(index.row(), index.column()) + return cat.data(idx) def flags(self, index): """Return the item flags for index. @@ -132,7 +134,7 @@ class CompletionModel(QAbstractItemModel): # categories have no parent return QModelIndex() row = self._categories.index(parent_cat) - return self.createIndex(row, 0, None) + return self.createIndex(row, index.column(), None) def rowCount(self, parent=QModelIndex()): """Override QAbstractItemModel::rowCount.""" diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index 75cc2642c..b22d795be 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -71,8 +71,7 @@ class ListCategory(QSortFilterProxyModel): parent: The parent item QModelIndex. Return: - True if self.pattern is contained in item, or if it's a root item - (category). False in all other cases + True if self.pattern is contained in item. """ if not self.pattern: return True diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index e2801b072..2060f383f 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -28,6 +28,7 @@ from qutebrowser.utils import debug class SqlCategory(QSqlQueryModel): + """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, @@ -57,12 +58,11 @@ class SqlCategory(QSqlQueryModel): if group_by: querystr += ' group by {}'.format(group_by) if sort_by: - assert sort_order in ['asc', 'desc'] + assert sort_order in ['asc', 'desc'], sort_order querystr += ' order by {} {}'.format(sort_by, sort_order) - self._query = sql.Query(querystr) + self._query = sql.Query(querystr, forward_only=False) self._param_count = len(filter_fields) - self.columns_to_filter = None # map filter_fields to indices col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name)) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 2c92815c9..7e42285a6 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -74,7 +74,8 @@ def url(): model.add_category(listcategory.ListCategory('Bookmarks', bookmarks, columns_to_filter=[0, 1])) - timefmt = config.get('completion', 'timestamp-format') + # replace 's to avoid breaking the query + timefmt = config.get('completion', 'timestamp-format').replace("'", "`") select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( 'CompletionHistory', sort_order='desc', sort_by='last_atime', diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index f174248b3..98eebba43 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -59,13 +59,23 @@ class Query(QSqlQuery): """A prepared SQL Query.""" - def __init__(self, querystr): + def __init__(self, querystr, forward_only=True): + """Prepare a new sql query. + + Args: + querystr: String to prepare query from. + forward_only: Optimization for queries that will only step forward. + Must be false for completion queries. + """ super().__init__(QSqlDatabase.database()) log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) - self.prepare(querystr) + if not self.prepare(querystr): + raise SqlException('Failed to prepare query "{}"'.format(querystr)) + self.setForwardOnly(forward_only) def __iter__(self): - assert self.isActive(), "Cannot iterate inactive query" + if not self.isActive(): + raise SqlException("Cannot iterate inactive query") rec = self.record() fields = [rec.fieldName(i) for i in range(rec.count())] rowtype = collections.namedtuple('ResultRow', fields) @@ -87,8 +97,8 @@ class Query(QSqlQuery): def value(self): """Return the result of a single-value query (e.g. an EXISTS).""" - ok = self.next() - assert ok, "No result for single-result query" + if not self.next(): + raise SqlException("No result for single-result query") return self.record().value(0) @@ -152,7 +162,7 @@ class SqlTable(QObject): Args: field: Field to match. """ - return Query("Select EXISTS(SELECT * FROM {} where {} = ?)" + return Query("SELECT EXISTS(SELECT * FROM {} WHERE {} = ?)" .format(self._name, field)) def __len__(self): @@ -214,17 +224,19 @@ class SqlTable(QObject): self.changed.emit() def delete_all(self): - """Remove all row from the table.""" + """Remove all rows from the table.""" Query("DELETE FROM {}".format(self._name)).run() self.changed.emit() def select(self, sort_by, sort_order, limit=-1): - """Remove all row from the table. + """Prepare, run, and return a select statement on this table. Args: sort_by: name of column to sort by. sort_order: 'asc' or 'desc'. limit: max number of rows in result, defaults to -1 (unlimited). + + Return: A prepared and executed select query. """ q = Query('SELECT * FROM {} ORDER BY {} {} LIMIT ?' .format(self._name, sort_by, sort_order)) diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index f6c5ce84a..1a1c0c95b 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -702,8 +702,6 @@ Feature: Various utility commands. And I wait for "Renderer process was killed" in the log And I open data/numbers/3.txt Then no crash should happen - And the following tabs should be open: - - data/numbers/3.txt (active) ## Other diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 197bc3d20..5249e891c 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -28,17 +28,16 @@ bdd.scenarios('history.feature') @bdd.then(bdd.parsers.parse("the history should contain:\n{expected}")) -def check_history(quteproc, expected, httpbin): - with tempfile.TemporaryDirectory() as tmpdir: - path = os.path.join(tmpdir, 'history') - quteproc.send_cmd(':debug-dump-history "{}"'.format(path)) - quteproc.wait_for(category='message', loglevel=logging.INFO, - message='Dumped history to {}.'.format(path)) +def check_history(quteproc, httpbin, tmpdir, expected): + path = tmpdir / 'history' + quteproc.send_cmd(':debug-dump-history "{}"'.format(path)) + quteproc.wait_for(category='message', loglevel=logging.INFO, + message='Dumped history to {}.'.format(path)) - with open(path, 'r', encoding='utf-8') as f: - # ignore access times, they will differ in each run - actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() - for line in f.read().splitlines()) + with open(path, 'r', encoding='utf-8') as f: + # ignore access times, they will differ in each run + actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() + for line in f.read()) expected = expected.replace('(port)', str(httpbin.port)) assert actual == expected diff --git a/tests/helpers/utils.py b/tests/helpers/utils.py index 7dbd7dd25..051b7bd50 100644 --- a/tests/helpers/utils.py +++ b/tests/helpers/utils.py @@ -170,3 +170,16 @@ def abs_datapath(): """Get the absolute path to the end2end data directory.""" file_abs = os.path.abspath(os.path.dirname(__file__)) return os.path.join(file_abs, '..', 'end2end', 'data') + + +def validate_model(cat, expected): + """Check that a category contains the expected items in the given order. + + Args: + cat: The category to inspect. + expected: A list of tuples containing the expected items. + """ + assert cat.rowCount() == len(expected) + for row, items in enumerate(expected): + for col, item in enumerate(items): + assert cat.data(cat.index(row, col)) == item diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 0f9f43373..87d0662b5 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -122,7 +122,6 @@ class TestHistoryHandler: url = QUrl("qute://history/data?start_time=" + str(start_time)) _mimetype, data = qutescheme.qute_history(url) items = json.loads(data) - items = [item for item in items if 'time' in item] # skip 'next' item assert len(items) == expected_item_count @@ -132,27 +131,6 @@ class TestHistoryHandler: assert item['time'] <= start_time * 1000 assert item['time'] > end_time * 1000 - @pytest.mark.skip("TODO: do we need next?") - @pytest.mark.parametrize("start_time_offset, next_time", [ - (0, 24*60*60), - (24*60*60, 48*60*60), - (48*60*60, -1), - (72*60*60, -1) - ]) - def test_qutehistory_next(self, start_time_offset, next_time, now): - """Ensure qute://history/data returns correct items.""" - start_time = now - start_time_offset - url = QUrl("qute://history/data?start_time=" + str(start_time)) - _mimetype, data = qutescheme.qute_history(url) - items = json.loads(data) - items = [item for item in items if 'next' in item] # 'next' items - assert len(items) == 1 - - if next_time == -1: - assert items[0]["next"] == -1 - else: - assert items[0]["next"] == now - next_time - def test_qute_history_benchmark(self, fake_web_history, benchmark, now): entries = [] for t in range(100000): # one history per second diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 5386dddd9..841da4ea7 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -107,14 +107,6 @@ def test_entries_before(hist): assert times == [12348, 12347, 12346] -def test_save(tmpdir, hist): - hist.add_url(QUrl('http://example.com/'), atime=12345) - hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890) - - hist2 = history.WebHistory() - assert list(hist2) == [('http://example.com/', '', 12345, False), - ('http://www.qutebrowser.org/', '', 67890, False)] - def test_clear(qtbot, tmpdir, hist, mocker): hist.add_url(QUrl('http://example.com/')) @@ -132,12 +124,11 @@ def test_clear_force(qtbot, tmpdir, hist): assert not len(hist) -@pytest.mark.parametrize('item', [ +@pytest.mark.parametrize('url, atime, title, redirect', [ ('http://www.example.com', 12346, 'the title', False), ('http://www.example.com', 12346, 'the title', True) ]) -def test_add_item(qtbot, hist, item): - (url, atime, title, redirect) = item +def test_add_item(qtbot, hist, url, atime, title, redirect): hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) assert list(hist) == [(url, title, atime, redirect)] @@ -188,13 +179,14 @@ def test_history_interface(qtbot, webview, hist_interface): def cleanup_init(): # prevent test_init from leaking state yield - try: - hist = objreg.get('web-history') + hist = objreg.get('web-history', None) + if hist is not None: hist.setParent(None) objreg.delete('web-history') + try: from PyQt5.QtWebKit import QWebHistoryInterface QWebHistoryInterface.setDefaultInterface(None) - except: + except Exception: pass diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 66f6e7f89..4cc59deea 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -229,7 +229,7 @@ def test_completion_item_del_no_selection(completionview): model = completionmodel.CompletionModel(delete_cur_item=func) model.add_category(listcategory.ListCategory('', [('foo',)])) completionview.set_model(model) - with pytest.raises(cmdexc.CommandError): + with pytest.raises(cmdexc.CommandError, match='No item selected!'): completionview.completion_item_del() assert not func.called @@ -240,5 +240,5 @@ def test_completion_item_del_no_func(completionview): model.add_category(listcategory.ListCategory('', [('foo',)])) completionview.set_model(model) completionview.completion_item_focus('next') - with pytest.raises(cmdexc.CommandError): + with pytest.raises(cmdexc.CommandError, match='Cannot delete this item.'): completionview.completion_item_del() diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index a0c300473..d8d35ea3c 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -21,22 +21,10 @@ import pytest +from helpers import utils from qutebrowser.completion.models import listcategory -def _validate(cat, expected): - """Check that a category contains the expected items in the given order. - - Args: - cat: The category to inspect. - expected: A list of tuples containing the expected items. - """ - assert cat.rowCount() == len(expected) - for row, items in enumerate(expected): - for col, item in enumerate(items): - assert cat.data(cat.index(row, col)) == item - - @pytest.mark.parametrize('pattern, filter_cols, before, after', [ ('foo', [0], [('foo', '', ''), ('bar', '', '')], @@ -68,4 +56,4 @@ def test_set_pattern(pattern, filter_cols, before, after): cat = listcategory.ListCategory('Foo', before, columns_to_filter=filter_cols) cat.set_pattern(pattern) - _validate(cat, after) + utils.validate_model(cat, after) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 1b4190d21..2a5b73445 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -21,6 +21,7 @@ import pytest +from helpers import utils from qutebrowser.misc import sql from qutebrowser.completion.models import sqlcategory @@ -28,19 +29,6 @@ from qutebrowser.completion.models import sqlcategory pytestmark = pytest.mark.usefixtures('init_sql') -def _validate(cat, expected): - """Check that a category contains the expected items in the given order. - - Args: - cat: The category to inspect. - expected: A list of tuples containing the expected items. - """ - assert cat.rowCount() == len(expected) - for row, items in enumerate(expected): - for col, item in enumerate(items): - assert cat.data(cat.index(row, col)) == item - - @pytest.mark.parametrize('sort_by, sort_order, data, expected', [ (None, 'asc', [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], @@ -77,7 +65,7 @@ def test_sorting(sort_by, sort_order, data, expected): cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) cat.set_pattern('') - _validate(cat, expected) + utils.validate_model(cat, expected) @pytest.mark.parametrize('pattern, filter_cols, before, after', [ @@ -133,7 +121,7 @@ def test_set_pattern(pattern, filter_cols, before, after): filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) cat.set_pattern(pattern) - _validate(cat, after) + utils.validate_model(cat, after) def test_select(): @@ -141,7 +129,7 @@ def test_select(): table.insert(['foo', 'bar', 'baz']) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') cat.set_pattern('') - _validate(cat, [('bar', 'baz', 'foo')]) + utils.validate_model(cat, [('bar', 'baz', 'foo')]) def test_where(): @@ -150,7 +138,7 @@ def test_where(): table.insert(['baz', 'biz', True]) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') cat.set_pattern('') - _validate(cat, [('foo', 'bar', False)]) + utils.validate_model(cat, [('foo', 'bar', False)]) def test_group(): @@ -162,7 +150,7 @@ def test_group(): cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='a, max(b)', group_by='a') cat.set_pattern('') - _validate(cat, [('bar', 3), ('foo', 2)]) + utils.validate_model(cat, [('bar', 3), ('foo', 2)]) def test_entry(): diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 11bf326fb..13dfb09d2 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -798,7 +798,6 @@ def test_chromium_version_unpatched(qapp): assert version._chromium_version() not in ['', 'unknown', 'unavailable'] -# pylint: disable=too-many-locals @pytest.mark.parametrize(['git_commit', 'frozen', 'style', 'with_webkit', 'known_distribution'], [ (True, False, True, True, True), # normal @@ -812,6 +811,7 @@ def test_chromium_version_unpatched(qapp): def test_version_output(git_commit, frozen, style, with_webkit, known_distribution, stubs, monkeypatch, init_sql): """Test version.version().""" + # pylint: disable=too-many-locals class FakeWebEngineProfile: def httpUserAgent(self): return 'Toaster/4.0.4 Chrome/CHROMIUMVERSION Teapot/4.1.8' From 9b0395db087f320e150626df7e182c66f9a47c86 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 8 Jun 2017 13:03:44 +0200 Subject: [PATCH 104/337] Add an lru cache for WebHistoryInterface.historyContains When loading heise.de, for some crazy reason QtWebKit calls historyContains about 16'000 times. With this cache (which we simply clear when *any* page has been loaded, as then the links which have been visited can change), that's down to 250 or so... --- qutebrowser/browser/webkit/webkithistory.py | 3 +++ qutebrowser/misc/utilcmds.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/qutebrowser/browser/webkit/webkithistory.py b/qutebrowser/browser/webkit/webkithistory.py index 64b7bf295..0edbb3fa3 100644 --- a/qutebrowser/browser/webkit/webkithistory.py +++ b/qutebrowser/browser/webkit/webkithistory.py @@ -19,6 +19,7 @@ """QtWebKit specific part of history.""" +import functools from PyQt5.QtWebKit import QWebHistoryInterface @@ -36,11 +37,13 @@ class WebHistoryInterface(QWebHistoryInterface): def __init__(self, webhistory, parent=None): super().__init__(parent) self._history = webhistory + self._history.changed.connect(self.historyContains.cache_clear) def addHistoryEntry(self, url_string): """Required for a QWebHistoryInterface impl, obsoleted by add_url.""" pass + @functools.lru_cache(maxsize=32768) def historyContains(self, url_string): """Called by WebKit to determine if a URL is contained in the history. diff --git a/qutebrowser/misc/utilcmds.py b/qutebrowser/misc/utilcmds.py index 41b44de1f..d1771c212 100644 --- a/qutebrowser/misc/utilcmds.py +++ b/qutebrowser/misc/utilcmds.py @@ -170,8 +170,15 @@ def debug_cache_stats(): """Print LRU cache stats.""" config_info = objreg.get('config').get.cache_info() style_info = style.get_stylesheet.cache_info() + try: + from PyQt5.QtWebKit import QWebHistoryInterface + interface = QWebHistoryInterface.defaultInterface() + history_info = interface.historyContains.cache_info() + except ImportError: + history_info = None log.misc.debug('config: {}'.format(config_info)) log.misc.debug('style: {}'.format(style_info)) + log.misc.debug('history: {}'.format(history_info)) @cmdutils.register(debug=True) From 3a4ef09f58312b3bc614150950ad662ebf92f2dd Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 07:34:35 -0400 Subject: [PATCH 105/337] More sql code review fixes --- qutebrowser/completion/completionwidget.py | 4 ++-- tests/end2end/features/test_history_bdd.py | 4 ++-- tests/unit/browser/webkit/test_history.py | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index f4d4d80b7..15f317c5a 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -149,7 +149,7 @@ class CompletionView(QTreeView): def _resize_columns(self): """Resize the completion columns based on column_widths.""" width = self.size().width() - column_widths = self.model.column_widths + column_widths = self.model().column_widths pixel_widths = [(width * perc // 100) for perc in column_widths] if self.verticalScrollBar().isVisible(): @@ -295,7 +295,7 @@ class CompletionView(QTreeView): def _maybe_show(self): if (config.get('completion', 'show') == 'always' and - model.count() > 0): + self.model().count() > 0): self.show() else: self.hide() diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 5249e891c..75f20efcb 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -44,5 +44,5 @@ def check_history(quteproc, httpbin, tmpdir, expected): @bdd.then("the history should be empty") -def check_history_empty(quteproc, httpbin): - check_history(quteproc, '', httpbin) +def check_history_empty(quteproc, httpbin, tmpdir): + check_history(quteproc, httpbin, tmpdir, '') diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 841da4ea7..49b499534 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -107,7 +107,6 @@ def test_entries_before(hist): assert times == [12348, 12347, 12346] - def test_clear(qtbot, tmpdir, hist, mocker): hist.add_url(QUrl('http://example.com/')) hist.add_url(QUrl('http://www.qutebrowser.org/')) @@ -186,7 +185,7 @@ def cleanup_init(): try: from PyQt5.QtWebKit import QWebHistoryInterface QWebHistoryInterface.setDefaultInterface(None) - except Exception: + except ImportError: pass From 18cd8ba0b6dc3bcfa68f77614fc2b59396968f6d Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 8 Jun 2017 13:58:36 +0200 Subject: [PATCH 106/337] Add indices for HistoryAtimeIndex and CompletionHistoryAtimeIndex before ------ sqlite> SELECT * FROM History where not redirect and not url like "qute://%" and atime > ? and atime <= ? ORDER BY atime desc; Run Time: real 0.072 user 0.063334 sys 0.010000 sqlite> explain query plan SELECT * FROM History where not redirect and not url like "qute://%" and atime > ? and atime <= ? ORDER BY atime desc; 0|0|0|SCAN TABLE History 0|0|0|USE TEMP B-TREE FOR ORDER BY sqlite> explain query plan select url, title, strftime('%Y-%m-%d', last_atime, 'unixepoch') from CompletionHistory where (url like "%qute%" or title like "%qute%") order by last_atime desc; 0|0|0|SCAN TABLE CompletionHistory 0|0|0|USE TEMP B-TREE FOR ORDER BY after ----- sqlite> SELECT * FROM History where not redirect and not url like "qute://%" and atime > ? and atime <= ? ORDER BY atime desc; Run Time: real 0.000 user 0.000000 sys 0.000000 sqlite> explain query plan SELECT * FROM History where not redirect and not url like "qute://%" and atime > ? and atime <= ? ORDER BY atime desc; 0|0|0|SEARCH TABLE History USING INDEX AtimeIndex (atime>? AND atime explain query plan select url, title, strftime('%Y-%m-%d', last_atime, 'unixepoch') from CompletionHistory where (url like "%qute%" or title like "%qute%") order by last_atime desc; 0|0|0|SCAN TABLE CompletionHistory USING INDEX CompletionAtimeIndex --- qutebrowser/browser/history.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 62c773866..05f2fefd5 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -79,6 +79,7 @@ class CompletionHistory(sql.SqlTable): def __init__(self, parent=None): super().__init__("CompletionHistory", ['url', 'title', 'last_atime'], constraints={'url': 'PRIMARY KEY'}, parent=parent) + self.create_index('CompletionHistoryAtimeIndex', 'last_atime') class WebHistory(sql.SqlTable): @@ -90,6 +91,7 @@ class WebHistory(sql.SqlTable): parent=parent) self.completion = CompletionHistory(parent=self) self.create_index('HistoryIndex', 'url') + self.create_index('HistoryAtimeIndex', 'atime') self._contains_query = self.contains_query('url') self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' From 862f8d318846a6dfdd49dc7a754ffb8c56be6289 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 08:43:53 -0400 Subject: [PATCH 107/337] Always return col 0 for index parent. This was changed during code review but was causing Qt errors while TAB-completing in the selection view: 08:42:34 WARNING qt Unknown module:none:0 Can't select indexes from different model or with different parents --- qutebrowser/completion/models/completionmodel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 2dec0ed46..2d74d94e0 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -134,7 +134,7 @@ class CompletionModel(QAbstractItemModel): # categories have no parent return QModelIndex() row = self._categories.index(parent_cat) - return self.createIndex(row, index.column(), None) + return self.createIndex(row, 0, None) def rowCount(self, parent=QModelIndex()): """Override QAbstractItemModel::rowCount.""" From 22b7b21d5a5d0d8db4243a3d8fd9a831f85c7cce Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 21:43:00 -0400 Subject: [PATCH 108/337] Use named placeholders for sql queries. --- qutebrowser/browser/history.py | 23 +++--- qutebrowser/completion/models/sqlcategory.py | 4 +- qutebrowser/misc/sql.py | 37 ++++++---- tests/unit/completion/test_sqlcategory.py | 18 ++--- tests/unit/misc/test_sql.py | 76 ++++++++++++-------- 5 files changed, 92 insertions(+), 66 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 05f2fefd5..0b9b7db81 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -96,30 +96,31 @@ class WebHistory(sql.SqlTable): self._between_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' - 'and atime > ? ' - 'and atime <= ? ' + 'and atime > :earliest ' + 'and atime <= :latest ' 'ORDER BY atime desc') self._before_query = sql.Query('SELECT * FROM History ' 'where not redirect ' 'and not url like "qute://%" ' - 'and atime <= ? ' + 'and atime <= :latest ' 'ORDER BY atime desc ' - 'limit ? offset ?') + 'limit :limit offset :offset') def __repr__(self): return utils.get_repr(self, length=len(self)) def __contains__(self, url): - return self._contains_query.run([url]).value() + return self._contains_query.run(val=url).value() def _add_entry(self, entry): """Add an entry to the in-memory database.""" - self.insert([entry.url_str(), entry.title, int(entry.atime), - entry.redirect]) + self.insert(url=entry.url_str(), title=entry.title, + atime=int(entry.atime), redirect=entry.redirect) if not entry.redirect: - self.completion.insert([entry.url_str(), entry.title, - int(entry.atime)], replace=True) + self.completion.insert_or_replace(url=entry.url_str(), + title=entry.title, + last_atime=int(entry.atime)) def get_recent(self): """Get the most recent history entries.""" @@ -132,7 +133,7 @@ class WebHistory(sql.SqlTable): earliest: Omit timestamps earlier than this. latest: Omit timestamps later than this. """ - self._between_query.run([earliest, latest]) + self._between_query.run(earliest=earliest, latest=latest) return iter(self._between_query) def entries_before(self, latest, limit, offset): @@ -143,7 +144,7 @@ class WebHistory(sql.SqlTable): limit: Max number of entries to include. offset: Number of entries to skip. """ - self._before_query.run([latest, limit, offset]) + self._before_query.run(latest=latest, limit=limit, offset=offset) return iter(self._before_query) @cmdutils.register(name='history-clear', instance='web-history') diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 2060f383f..d49b1db51 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -49,7 +49,7 @@ class SqlCategory(QSqlQueryModel): querystr = 'select {} from {} where ('.format(select, name) # the incoming pattern will have literal % and _ escaped with '\' # we need to tell sql to treat '\' as an escape character - querystr += ' or '.join("{} like ? escape '\\'".format(f) + querystr += ' or '.join("{} like :pattern escape '\\'".format(f) for f in filter_fields) querystr += ')' @@ -83,5 +83,5 @@ class SqlCategory(QSqlQueryModel): pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) with debug.log_time('sql', 'Running completion query'): - self._query.run([pattern] * self._param_count) + self._query.run(pattern=pattern) self.setQuery(self._query) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 98eebba43..5c3e94c1e 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -84,11 +84,11 @@ class Query(QSqlQuery): rec = self.record() yield rowtype(*[rec.value(i) for i in range(rec.count())]) - def run(self, values=None): + def run(self, **values): """Execute the prepared query.""" log.sql.debug('Running SQL query: "{}"'.format(self.lastQuery())) - for val in values or []: - self.addBindValue(val) + for key, val in values.items(): + self.bindValue(':{}'.format(key), val) log.sql.debug('self bindings: {}'.format(self.boundValues())) if not self.exec_(): raise SqlException('Failed to exec query "{}": "{}"'.format( @@ -162,7 +162,7 @@ class SqlTable(QObject): Args: field: Field to match. """ - return Query("SELECT EXISTS(SELECT * FROM {} WHERE {} = ?)" + return Query("SELECT EXISTS(SELECT * FROM {} WHERE {} = :val)" .format(self._name, field)) def __len__(self): @@ -181,23 +181,34 @@ class SqlTable(QObject): Return: The number of rows deleted. """ - q = Query("DELETE FROM {} where {} = ?".format(self._name, field)) - q.run([value]) + q = Query("DELETE FROM {} where {} = :val".format(self._name, field)) + q.run(val=value) if not q.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, values, replace=False): + def insert(self, **values): """Append a row to the table. Args: values: A list of values to insert. replace: If set, replace existing values. """ - paramstr = ','.join(['?'] * len(values)) - q = Query("INSERT {} INTO {} values({})".format( - 'OR REPLACE' if replace else '', self._name, paramstr)) - q.run(values) + paramstr = ','.join(':{}'.format(key) for key in values.keys()) + q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q.run(**values) + self.changed.emit() + + def insert_or_replace(self, **values): + """Append a row to the table. + + Args: + values: A list of values to insert. + replace: If set, replace existing values. + """ + paramstr = ','.join(':{}'.format(key) for key in values.keys()) + q = Query("REPLACE INTO {} values({})".format(self._name, paramstr)) + q.run(**values) self.changed.emit() def insert_batch(self, rows, replace=False): @@ -238,7 +249,7 @@ class SqlTable(QObject): Return: A prepared and executed select query. """ - q = Query('SELECT * FROM {} ORDER BY {} {} LIMIT ?' + q = Query('SELECT * FROM {} ORDER BY {} {} LIMIT :limit' .format(self._name, sort_by, sort_order)) - q.run([limit]) + q.run(limit=limit) return q diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 2a5b73445..cef17c964 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -61,7 +61,7 @@ pytestmark = pytest.mark.usefixtures('init_sql') def test_sorting(sort_by, sort_order, data, expected): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: - table.insert(row) + table.insert(a=row[0], b=row[1], c=row[2]) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) cat.set_pattern('') @@ -117,7 +117,7 @@ def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: - table.insert(row) + table.insert(a=row[0], b=row[1], c=row[2]) filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) cat.set_pattern(pattern) @@ -126,7 +126,7 @@ def test_set_pattern(pattern, filter_cols, before, after): def test_select(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert(['foo', 'bar', 'baz']) + table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') cat.set_pattern('') utils.validate_model(cat, [('bar', 'baz', 'foo')]) @@ -134,8 +134,8 @@ def test_select(): def test_where(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert(['foo', 'bar', False]) - table.insert(['baz', 'biz', True]) + table.insert({'a': 'foo', 'b': 'bar', 'c': False}) + table.insert({'a': 'baz', 'b': 'biz', 'c': True}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') cat.set_pattern('') utils.validate_model(cat, [('foo', 'bar', False)]) @@ -143,10 +143,10 @@ def test_where(): def test_group(): table = sql.SqlTable('Foo', ['a', 'b']) - table.insert(['foo', 1]) - table.insert(['bar', 3]) - table.insert(['foo', 2]) - table.insert(['bar', 0]) + table.insert({'a': 'foo', 'b': 1}) + table.insert({'a': 'bar', 'b': 3}) + table.insert({'a': 'foo', 'b': 2}) + table.insert({'a': 'bar', 'b': 0}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='a, max(b)', group_by='a') cat.set_pattern('') diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 03cb0f27c..3216400f0 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -35,39 +35,53 @@ def test_init(): def test_insert(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) with qtbot.waitSignal(table.changed): - table.insert(['one', 1, False]) + table.insert(name='one', val=1, lucky=False) with qtbot.waitSignal(table.changed): - table.insert(['wan', 1, False]) + table.insert(name='wan', val=1, lucky=False) + + +def test_insert_or_replace(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], + constraints={'name': 'PRIMARY KEY'}) + with qtbot.waitSignal(table.changed): + table.insert_or_replace(name='one', val=1, lucky=False) + with qtbot.waitSignal(table.changed): + table.insert_or_replace(name='one', val=11, lucky=True) + assert list(table) == [('one', 11, True)] def test_iter(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) + table.insert(name='one', val=1, lucky=False) + table.insert(name='nine', val=9, lucky=False) + table.insert(name='thirteen', val=13, lucky=True) assert list(table) == [('one', 1, False), ('nine', 9, False), ('thirteen', 13, True)] @pytest.mark.parametrize('rows, sort_by, sort_order, limit, result', [ - ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', 5, [(1, 6), (2, 5), (3, 4)]), - ([[2, 5], [1, 6], [3, 4]], 'a', 'desc', 3, [(3, 4), (2, 5), (1, 6)]), - ([[2, 5], [1, 6], [3, 4]], 'b', 'desc', 2, [(1, 6), (2, 5)]), - ([[2, 5], [1, 6], [3, 4]], 'a', 'asc', -1, [(1, 6), (2, 5), (3, 4)]), + ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'a', 'asc', 5, + [(1, 6), (2, 5), (3, 4)]), + ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'a', 'desc', 3, + [(3, 4), (2, 5), (1, 6)]), + ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'b', 'desc', 2, + [(1, 6), (2, 5)]), + ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'a', 'asc', -1, + [(1, 6), (2, 5), (3, 4)]), ]) def test_select(rows, sort_by, sort_order, limit, result): table = sql.SqlTable('Foo', ['a', 'b']) for row in rows: - table.insert(row) + table.insert(**row) assert list(table.select(sort_by, sort_order, limit)) == result def test_delete(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) + table.insert(name='one', val=1, lucky=False) + table.insert(name='nine', val=9, lucky=False) + table.insert(name='thirteen', val=13, lucky=True) with pytest.raises(KeyError): table.delete('nope', 'name') with qtbot.waitSignal(table.changed): @@ -81,40 +95,40 @@ def test_delete(qtbot): def test_len(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) assert len(table) == 0 - table.insert(['one', 1, False]) + table.insert(name='one', val=1, lucky=False) assert len(table) == 1 - table.insert(['nine', 9, False]) + table.insert(name='nine', val=9, lucky=False) assert len(table) == 2 - table.insert(['thirteen', 13, True]) + table.insert(name='thirteen', val=13, lucky=True) assert len(table) == 3 def test_contains(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) + table.insert(name='one', val=1, lucky=False) + table.insert(name='nine', val=9, lucky=False) + table.insert(name='thirteen', val=13, lucky=True) name_query = table.contains_query('name') val_query = table.contains_query('val') lucky_query = table.contains_query('lucky') - assert name_query.run(['one']).value() - assert name_query.run(['thirteen']).value() - assert val_query.run([9]).value() - assert lucky_query.run([False]).value() - assert lucky_query.run([True]).value() - assert not name_query.run(['oone']).value() - assert not name_query.run([1]).value() - assert not name_query.run(['*']).value() - assert not val_query.run([10]).value() + assert name_query.run(val='one').value() + assert name_query.run(val='thirteen').value() + assert val_query.run(val=9).value() + assert lucky_query.run(val=False).value() + assert lucky_query.run(val=True).value() + assert not name_query.run(val='oone').value() + assert not name_query.run(val=1).value() + assert not name_query.run(val='*').value() + assert not val_query.run(val=10).value() def test_delete_all(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(['one', 1, False]) - table.insert(['nine', 9, False]) - table.insert(['thirteen', 13, True]) + table.insert(name='one', val=1, lucky=False) + table.insert(name='nine', val=9, lucky=False) + table.insert(name='thirteen', val=13, lucky=True) with qtbot.waitSignal(table.changed): table.delete_all() assert list(table) == [] From a6a9ad72f91e9401c37535045a320da305dc48e9 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 22:18:34 -0400 Subject: [PATCH 109/337] Fix test_history_interface. This was still using a history dict instead of SQL history. --- tests/unit/browser/webkit/test_history.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 49b499534..f99df5587 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -152,7 +152,7 @@ def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog): @pytest.fixture -def hist_interface(): +def hist_interface(hist): # pylint: disable=invalid-name QtWebKit = pytest.importorskip('PyQt5.QtWebKit') from qutebrowser.browser.webkit import webkithistory @@ -160,8 +160,7 @@ def hist_interface(): # pylint: enable=invalid-name entry = history.Entry(atime=0, url=QUrl('http://www.example.com/'), title='example') - history_dict = {'http://www.example.com/': entry} - interface = webkithistory.WebHistoryInterface(history_dict) + interface = webkithistory.WebHistoryInterface(hist) QWebHistoryInterface.setDefaultInterface(interface) yield QWebHistoryInterface.setDefaultInterface(None) From fa39b82b3c7d6ffa5f04453b3f1bffe7811f15d4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 22:18:52 -0400 Subject: [PATCH 110/337] Backup old history file after import. Instead of removing it, move it to history.bak. --- qutebrowser/browser/history.py | 10 ++++++---- tests/unit/browser/webkit/test_history.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 0b9b7db81..4739e56c8 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -240,7 +240,8 @@ class WebHistory(sql.SqlTable): """Import a history text file into sqlite if it exists. In older versions of qutebrowser, history was stored in a text format. - This converts that file into the new sqlite format and removes it. + This converts that file into the new sqlite format and moves it to a + backup location. """ path = os.path.join(standarddir.data(), 'history') if not os.path.isfile(path): @@ -253,9 +254,10 @@ class WebHistory(sql.SqlTable): except ValueError as ex: message.error('Failed to import history: {}'.format(ex)) else: - message.info('History import complete. Removing {}' - .format(path)) - os.remove(path) + bakpath = path + '.bak' + message.info('History import complete. Moving {} to {}' + .format(path, bakpath)) + os.rename(path, bakpath) # delay to give message time to appear before locking down for import message.info('Converting {} to sqlite...'.format(path)) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index f99df5587..d87f30b38 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -240,6 +240,7 @@ def test_read(hist, data_tmpdir, monkeypatch, stubs): ] assert not histfile.exists() + assert (data_tmpdir / 'history.bak').exists() @pytest.mark.parametrize('line', [ From 61a1709141013a60c159d99b1b4919d611919be1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 8 Jun 2017 22:27:46 -0400 Subject: [PATCH 111/337] Fix completion selection bug. Fix the issue where pressing `oo` would show a url completion dialog where attempting to select items would do nothing but show a Qt warning. The fix is to ensure we set _last_completion_func to None whenever we clear completion (there was a case I missed). It also ensures we always delete the old model and adds a safety to prevent deleting an in-use model is set_model is called with the current model. --- qutebrowser/completion/completer.py | 1 + qutebrowser/completion/completionwidget.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 491fe2e04..ae72add20 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -211,6 +211,7 @@ class Completer(QObject): # FIXME complete searches # https://github.com/qutebrowser/qutebrowser/issues/32 completion.set_model(None) + self._last_completion_func = None return before_cursor, pattern, after_cursor = self._partition() diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 15f317c5a..7ea4edf13 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -269,17 +269,18 @@ class CompletionView(QTreeView): Args: model: The model to use. """ + if self.model() is not None and model is not self.model(): + self.model().deleteLater() + self.selectionModel().deleteLater() + + self.setModel(model) + if model is None: self._active = False self.hide() return - if self.model() is not None: - self.model().deleteLater() - self.selectionModel().deleteLater() - model.setParent(self) - self.setModel(model) self._active = True self._maybe_show() From 679e001a48d763b8fed7788ad4f4ccef6900d906 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 9 Jun 2017 08:06:12 -0400 Subject: [PATCH 112/337] Separate sqlcategory title from table name. Also fix a number of sql/completion tests that were failing. --- qutebrowser/completion/models/sqlcategory.py | 10 +++++---- qutebrowser/completion/models/urlmodel.py | 3 ++- tests/unit/completion/test_models.py | 23 +++++++++----------- tests/unit/completion/test_sqlcategory.py | 14 ++++++------ 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index d49b1db51..7e0ecf2c5 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -31,12 +31,14 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" - def __init__(self, name, *, filter_fields, sort_by=None, sort_order=None, - select='*', where=None, group_by=None, parent=None): + def __init__(self, name, *, title=None, filter_fields, sort_by=None, + sort_order=None, select='*', where=None, group_by=None, + parent=None): """Create a new completion category backed by a sql table. Args: - name: Name of category, and the table in the database. + name: Name of the table in the database. + title: Title of category, defaults to table name. filter_fields: Names of fields to apply filter pattern to. select: A custom result column expression for the select statement. where: An optional clause to filter out some rows. @@ -44,7 +46,7 @@ class SqlCategory(QSqlQueryModel): sort_order: Either 'asc' or 'desc', if sort_by is non-None """ super().__init__(parent=parent) - self.name = name + self.name = title or name querystr = 'select {} from {} where ('.format(select, name) # the incoming pattern will have literal % and _ escaped with '\' diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 7e42285a6..8cacca873 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -78,7 +78,8 @@ def url(): timefmt = config.get('completion', 'timestamp-format').replace("'", "`") select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) hist_cat = sqlcategory.SqlCategory( - 'CompletionHistory', sort_order='desc', sort_by='last_atime', + 'CompletionHistory', title='History', + sort_order='desc', sort_by='last_atime', filter_fields=['url', 'title'], select='url, title, {}'.format(select_time)) model.add_category(hist_cat) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index c85ecbf38..89da30013 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -161,16 +161,15 @@ def web_history_stub(stubs, init_sql): @pytest.fixture def web_history(web_history_stub, init_sql): """Pre-populate the web-history database.""" - web_history_stub.insert(['http://qutebrowser.org', 'qutebrowser', - datetime(2015, 9, 5).timestamp()]) - web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2016, 2, 8).timestamp()]) - web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2016, 3, 8).timestamp()]) - web_history_stub.insert(['https://python.org', 'Welcome to Python.org', - datetime(2014, 3, 8).timestamp()]) - web_history_stub.insert(['https://github.com', 'https://github.com', - datetime(2016, 5, 1).timestamp()]) + web_history_stub.insert(url='http://qutebrowser.org', + title='qutebrowser', + last_atime=datetime(2015, 9, 5).timestamp()) + web_history_stub.insert(url='https://python.org', + title='Welcome to Python.org', + last_atime=datetime(2016, 3, 8).timestamp()) + web_history_stub.insert(url='https://github.com', + title='https://github.com', + last_atime=datetime(2016, 5, 1).timestamp()) return web_history_stub @@ -304,9 +303,7 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, "History": [ ('https://github.com', 'https://github.com', '2016-05-01'), ('https://python.org', 'Welcome to Python.org', '2016-03-08'), - ('https://python.org', 'Welcome to Python.org', '2016-02-08'), ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), - ('https://python.org', 'Welcome to Python.org', '2014-03-08'), ], }) @@ -334,7 +331,7 @@ def test_url_completion_pattern(config_stub, web_history_stub, url, title, pattern, rowcount): """Test that url completion filters by url and title.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} - web_history_stub.insert([url, title, 0]) + web_history_stub.insert(url=url, title=title, last_atime=0) model = urlmodel.url() model.set_pattern(pattern) # 2, 0 is History diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index cef17c964..d81fa8cf0 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -126,7 +126,7 @@ def test_set_pattern(pattern, filter_cols, before, after): def test_select(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) + table.insert(a='foo', b='bar', c='baz') cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') cat.set_pattern('') utils.validate_model(cat, [('bar', 'baz', 'foo')]) @@ -134,8 +134,8 @@ def test_select(): def test_where(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert({'a': 'foo', 'b': 'bar', 'c': False}) - table.insert({'a': 'baz', 'b': 'biz', 'c': True}) + table.insert(a='foo', b='bar', c=False) + table.insert(a='baz', b='biz', c=True) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') cat.set_pattern('') utils.validate_model(cat, [('foo', 'bar', False)]) @@ -143,10 +143,10 @@ def test_where(): def test_group(): table = sql.SqlTable('Foo', ['a', 'b']) - table.insert({'a': 'foo', 'b': 1}) - table.insert({'a': 'bar', 'b': 3}) - table.insert({'a': 'foo', 'b': 2}) - table.insert({'a': 'bar', 'b': 0}) + table.insert(a='foo', b=1) + table.insert(a='bar', b=3) + table.insert(a='foo', b=2) + table.insert(a='bar', b=0) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='a, max(b)', group_by='a') cat.set_pattern('') From cf23f42b99f99c170c2a42511aabf53d6b51ae82 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 9 Jun 2017 12:07:39 -0400 Subject: [PATCH 113/337] Use splitlines in test_history_bdd again. Just using read() returns a single string, and iterating over that iterates over each character. --- tests/end2end/features/test_history_bdd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 75f20efcb..ce9faeca9 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -37,7 +37,7 @@ def check_history(quteproc, httpbin, tmpdir, expected): with open(path, 'r', encoding='utf-8') as f: # ignore access times, they will differ in each run actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() - for line in f.read()) + for line in f.read().splitlines()) expected = expected.replace('(port)', str(httpbin.port)) assert actual == expected From 1fe18134319e59763bfad93f8e43f8568cf87f3e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 10 Jun 2017 07:53:52 -0400 Subject: [PATCH 114/337] Fix pylint errors. --- qutebrowser/app.py | 2 +- qutebrowser/completion/models/listcategory.py | 2 +- qutebrowser/completion/models/sqlcategory.py | 1 - qutebrowser/misc/sql.py | 4 ++-- tests/end2end/features/test_history_bdd.py | 2 -- tests/unit/browser/webkit/test_history.py | 3 +-- 6 files changed, 5 insertions(+), 9 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 8f7984f2d..a8fd7ad54 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -57,7 +57,7 @@ from qutebrowser.misc import (readline, ipc, savemanager, sessions, crashsignal, earlyinit, objects, sql) from qutebrowser.misc import utilcmds # pylint: disable=unused-import from qutebrowser.utils import (log, version, message, utils, qtutils, urlutils, - objreg, usertypes, standarddir, error, debug) + objreg, usertypes, standarddir, error) # We import utilcmds to run the cmdutils.register decorators. diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index b22d795be..acdc5cafc 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -28,7 +28,7 @@ import re from PyQt5.QtCore import QSortFilterProxyModel from PyQt5.QtGui import QStandardItem, QStandardItemModel -from qutebrowser.utils import qtutils, debug, log +from qutebrowser.utils import qtutils class ListCategory(QSortFilterProxyModel): diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 7e0ecf2c5..9e88b52da 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -64,7 +64,6 @@ class SqlCategory(QSqlQueryModel): querystr += ' order by {} {}'.format(sort_by, sort_order) self._query = sql.Query(querystr, forward_only=False) - self._param_count = len(filter_fields) # map filter_fields to indices col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name)) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 5c3e94c1e..27b4d4268 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -194,7 +194,7 @@ class SqlTable(QObject): values: A list of values to insert. replace: If set, replace existing values. """ - paramstr = ','.join(':{}'.format(key) for key in values.keys()) + paramstr = ','.join(':{}'.format(key) for key in values) q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) q.run(**values) self.changed.emit() @@ -206,7 +206,7 @@ class SqlTable(QObject): values: A list of values to insert. replace: If set, replace existing values. """ - paramstr = ','.join(':{}'.format(key) for key in values.keys()) + paramstr = ','.join(':{}'.format(key) for key in values) q = Query("REPLACE INTO {} values({})".format(self._name, paramstr)) q.run(**values) self.changed.emit() diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index ce9faeca9..c1000d6ce 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -18,9 +18,7 @@ # along with qutebrowser. If not, see . import logging -import os.path import re -import tempfile import pytest_bdd as bdd diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index d87f30b38..04081e7c7 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -158,8 +158,7 @@ def hist_interface(hist): from qutebrowser.browser.webkit import webkithistory QWebHistoryInterface = QtWebKit.QWebHistoryInterface # pylint: enable=invalid-name - entry = history.Entry(atime=0, url=QUrl('http://www.example.com/'), - title='example') + hist.add_url(url=QUrl('http://www.example.com/'), title='example') interface = webkithistory.WebHistoryInterface(hist) QWebHistoryInterface.setDefaultInterface(interface) yield From f4f52ee204513f0c4d0c9a596977a2f6c00494e6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 10 Jun 2017 13:51:11 -0400 Subject: [PATCH 115/337] Remove history.Entry. No longer needed with sql backend. Query results build their own namedtuple from the returned columns, and inserting new entries is just done with named parameters. --- qutebrowser/browser/history.py | 62 +++------------------------ tests/unit/browser/test_qutescheme.py | 7 +-- 2 files changed, 11 insertions(+), 58 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 4739e56c8..6965cf3ea 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -30,48 +30,6 @@ from qutebrowser.utils import (utils, objreg, log, qtutils, usertypes, message, from qutebrowser.misc import objects, sql -class Entry: - - """A single entry in the web history. - - Attributes: - atime: The time the page was accessed. - url: The URL which was accessed as QUrl. - redirect: If True, don't show this entry in completion - """ - - def __init__(self, atime, url, title, redirect=False): - self.atime = float(atime) - self.url = url - self.title = title - self.redirect = redirect - qtutils.ensure_valid(url) - - def __repr__(self): - return utils.get_repr(self, constructor=True, atime=self.atime, - url=self.url_str(), title=self.title, - redirect=self.redirect) - - def __str__(self): - atime = str(int(self.atime)) - if self.redirect: - atime += '-r' # redirect flag - elems = [atime, self.url_str()] - if self.title: - elems.append(self.title) - return ' '.join(elems) - - def __eq__(self, other): - return (self.atime == other.atime and - self.title == other.title and - self.url == other.url and - self.redirect == other.redirect) - - def url_str(self): - """Get the URL as a lossless string.""" - return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) - - class CompletionHistory(sql.SqlTable): """History which only has the newest entry for each URL.""" @@ -113,15 +71,6 @@ class WebHistory(sql.SqlTable): def __contains__(self, url): return self._contains_query.run(val=url).value() - def _add_entry(self, entry): - """Add an entry to the in-memory database.""" - self.insert(url=entry.url_str(), title=entry.title, - atime=int(entry.atime), redirect=entry.redirect) - if not entry.redirect: - self.completion.insert_or_replace(url=entry.url_str(), - title=entry.title, - last_atime=int(entry.atime)) - def get_recent(self): """Get the most recent history entries.""" return self.select(sort_by='atime', sort_order='desc', limit=100) @@ -199,10 +148,13 @@ class WebHistory(sql.SqlTable): log.misc.warning("Ignoring invalid URL being added to history") return - if atime is None: - atime = time.time() - entry = Entry(atime, url, title, redirect=redirect) - self._add_entry(entry) + atime = int(atime) if (atime is not None) else int(time.time()) + url_str = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) + self.insert(url=url_str, title=title, atime=atime, redirect=redirect) + if not redirect: + self.completion.insert_or_replace(url=url_str, + title=title, + last_atime=atime) def _parse_entry(self, line): """Parse a history line like '12345 http://example.com title'.""" diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 87d0662b5..fefafc9a1 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -89,8 +89,9 @@ class TestHistoryHandler: items = [] for i in range(entry_count): entry_atime = now - i * interval - entry = history.Entry(atime=str(entry_atime), - url=QUrl("www.x.com/" + str(i)), title="Page " + str(i)) + entry = {"atime": str(entry_atime), + "url": QUrl("www.x.com/" + str(i)), + "title": "Page " + str(i)} items.insert(0, entry) return items @@ -107,7 +108,7 @@ class TestHistoryHandler: def fake_history(self, fake_web_history, entries): """Create fake history.""" for item in entries: - fake_web_history._add_entry(item) + fake_web_history.add_url(**item) @pytest.mark.parametrize("start_time_offset, expected_item_count", [ (0, 4), From e436f4816493de4314a0f48fa256096bdb0aa1f4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 10 Jun 2017 18:41:24 -0400 Subject: [PATCH 116/337] Small sql fixes. - Remove unused SqlTable.Entry - Fix wording of two log messages - Remove unused import --- qutebrowser/browser/history.py | 4 ++-- qutebrowser/misc/sql.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 6965cf3ea..75c184198 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -25,7 +25,7 @@ import time from PyQt5.QtCore import pyqtSlot, QUrl, QTimer from qutebrowser.commands import cmdutils, cmdexc -from qutebrowser.utils import (utils, objreg, log, qtutils, usertypes, message, +from qutebrowser.utils import (utils, objreg, log, usertypes, message, debug, standarddir) from qutebrowser.misc import objects, sql @@ -257,7 +257,7 @@ class WebHistory(sql.SqlTable): f.write('\n'.join(lines)) except OSError as e: raise cmdexc.CommandError('Could not write history: {}', e) - message.info("Dumped history to {}.".format(dest)) + message.info("Dumped history to {}".format(dest)) def init(parent=None): diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 27b4d4268..301ffb2ff 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -89,7 +89,7 @@ class Query(QSqlQuery): log.sql.debug('Running SQL query: "{}"'.format(self.lastQuery())) for key, val in values.items(): self.bindValue(':{}'.format(key), val) - log.sql.debug('self bindings: {}'.format(self.boundValues())) + log.sql.debug('query bindings: {}'.format(self.boundValues())) if not self.exec_(): raise SqlException('Failed to exec query "{}": "{}"'.format( self.lastQuery(), self.lastError().text())) @@ -107,7 +107,6 @@ class SqlTable(QObject): """Interface to a sql table. Attributes: - Entry: The class wrapping row data from this table. _name: Name of the SQL table this wraps. Signals: @@ -136,8 +135,6 @@ class SqlTable(QObject): .format(name, ','.join(column_defs))) q.run() - # pylint: disable=invalid-name - self.Entry = collections.namedtuple(name + '_Entry', fields) def create_index(self, name, field): """Create an index over this table. From 4e87773d895b4c14e52231c0b4f997ea9e7933d3 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 11 Jun 2017 13:57:38 -0400 Subject: [PATCH 117/337] Use a dict instead of named params for insert. This allows replace to be a named parameter and allows consolidating some duplicate code between various insert methods. This also fixes some tests that broke because batch insert was broken. --- qutebrowser/browser/history.py | 38 ++++++------ qutebrowser/misc/sql.py | 38 +++++------- tests/unit/browser/test_qutescheme.py | 15 +++-- tests/unit/completion/test_models.py | 43 +++++++------- tests/unit/misc/test_sql.py | 83 ++++++++++++++++++++------- 5 files changed, 126 insertions(+), 91 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 75c184198..e314f6c5e 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -150,11 +150,15 @@ class WebHistory(sql.SqlTable): atime = int(atime) if (atime is not None) else int(time.time()) url_str = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword) - self.insert(url=url_str, title=title, atime=atime, redirect=redirect) + self.insert({'url': url_str, + 'title': title, + 'atime': atime, + 'redirect': redirect}) if not redirect: - self.completion.insert_or_replace(url=url_str, - title=title, - last_atime=atime) + self.completion.insert({'url': url_str, + 'title': title, + 'last_atime': atime}, + replace=True) def _parse_entry(self, line): """Parse a history line like '12345 http://example.com title'.""" @@ -183,10 +187,7 @@ class WebHistory(sql.SqlTable): raise ValueError("Invalid flags {!r}".format(flags)) redirect = 'r' in flags - row = (url, title, float(atime), redirect) - completion_row = None if redirect else (url, title, float(atime)) - - return (row, completion_row) + return (url, title, int(atime), redirect) def import_txt(self): """Import a history text file into sqlite if it exists. @@ -218,22 +219,27 @@ class WebHistory(sql.SqlTable): def _read(self, path): """Import a text file into the sql database.""" with open(path, 'r', encoding='utf-8') as f: - rows = [] - completion_rows = [] + data = {'url': [], 'title': [], 'atime': [], 'redirect': []} + completion_data = {'url': [], 'title': [], 'last_atime': []} for (i, line) in enumerate(f): line = line.strip() if not line: continue try: - row, completion_row = self._parse_entry(line.strip()) - rows.append(row) - if completion_row is not None: - completion_rows.append(completion_row) + url, title, atime, redirect = self._parse_entry(line) + data['url'].append(url) + data['title'].append(title) + data['atime'].append(atime) + data['redirect'].append(redirect) + if not redirect: + completion_data['url'].append(url) + completion_data['title'].append(title) + completion_data['last_atime'].append(atime) except ValueError as ex: raise ValueError('Failed to parse line #{} of {}: "{}"' .format(i, path, ex)) - self.insert_batch(rows) - self.completion.insert_batch(completion_rows, replace=True) + self.insert_batch(data) + self.completion.insert_batch(completion_data, replace=True) @cmdutils.register(instance='web-history', debug=True) def debug_dump_history(self, dest): diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 301ffb2ff..717ae3c0f 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -184,44 +184,32 @@ class SqlTable(QObject): raise KeyError('No row with {} = "{}"'.format(field, value)) self.changed.emit() - def insert(self, **values): + def _insert_query(self, values, replace): + params = ','.join(':{}'.format(key) for key in values) + verb = "REPLACE" if replace else "INSERT" + return Query("{} INTO {} values({})".format(verb, self._name, params)) + + def insert(self, values, replace=False): """Append a row to the table. Args: - values: A list of values to insert. + values: A dict with a value to insert for each field name. replace: If set, replace existing values. """ - paramstr = ','.join(':{}'.format(key) for key in values) - q = Query("INSERT INTO {} values({})".format(self._name, paramstr)) + q = self._insert_query(values, replace) q.run(**values) self.changed.emit() - def insert_or_replace(self, **values): - """Append a row to the table. - - Args: - values: A list of values to insert. - replace: If set, replace existing values. - """ - paramstr = ','.join(':{}'.format(key) for key in values) - q = Query("REPLACE INTO {} values({})".format(self._name, paramstr)) - q.run(**values) - self.changed.emit() - - def insert_batch(self, rows, replace=False): + def insert_batch(self, values, replace=False): """Performantly append multiple rows to the table. Args: rows: A list of lists, where each sub-list is a row. - replace: If set, replace existing values. + values: A dict with a list of values to insert for each field name. """ - paramstr = ','.join(['?'] * len(rows[0])) - q = Query("INSERT {} INTO {} values({})".format( - 'OR REPLACE' if replace else '', self._name, paramstr)) - - transposed = [list(row) for row in zip(*rows)] - for val in transposed: - q.addBindValue(val) + q = self._insert_query(values, replace) + for key, val in values.items(): + q.bindValue(':{}'.format(key), val) db = QSqlDatabase.database() db.transaction() diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index fefafc9a1..154af0355 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -133,14 +133,13 @@ class TestHistoryHandler: assert item['time'] > end_time * 1000 def test_qute_history_benchmark(self, fake_web_history, benchmark, now): - entries = [] - for t in range(100000): # one history per second - entry = fake_web_history.Entry( - atime=str(now - t), - url=QUrl('www.x.com/{}'.format(t)), - title='x at {}'.format(t), - redirect=False) - entries.append(entry) + r = range(100000) + entries = { + 'atime': [int(now - t) for t in r], + 'url': ['www.x.com/{}'.format(t) for t in r], + 'title': ['x at {}'.format(t) for t in r], + 'redirect': [False for _ in r], + } fake_web_history.insert_batch(entries) url = QUrl("qute://history/data?start_time={}".format(now)) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 89da30013..e9e8e6366 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -161,15 +161,15 @@ def web_history_stub(stubs, init_sql): @pytest.fixture def web_history(web_history_stub, init_sql): """Pre-populate the web-history database.""" - web_history_stub.insert(url='http://qutebrowser.org', - title='qutebrowser', - last_atime=datetime(2015, 9, 5).timestamp()) - web_history_stub.insert(url='https://python.org', - title='Welcome to Python.org', - last_atime=datetime(2016, 3, 8).timestamp()) - web_history_stub.insert(url='https://github.com', - title='https://github.com', - last_atime=datetime(2016, 5, 1).timestamp()) + web_history_stub.insert({'url': 'http://qutebrowser.org', + 'title': 'qutebrowser', + 'last_atime': datetime(2015, 9, 5).timestamp()}) + web_history_stub.insert({'url': 'https://python.org', + 'title': 'Welcome to Python.org', + 'last_atime': datetime(2016, 3, 8).timestamp()}) + web_history_stub.insert({'url': 'https://github.com', + 'title': 'https://github.com', + 'last_atime': datetime(2016, 5, 1).timestamp()}) return web_history_stub @@ -331,7 +331,7 @@ def test_url_completion_pattern(config_stub, web_history_stub, url, title, pattern, rowcount): """Test that url completion filters by url and title.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} - web_history_stub.insert(url=url, title=title, last_atime=0) + web_history_stub.insert({'url': url, 'title': title, 'last_atime': 0}) model = urlmodel.url() model.set_pattern(pattern) # 2, 0 is History @@ -576,21 +576,22 @@ def test_url_completion_benchmark(benchmark, config_stub, config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 1000} - entries = [web_history_stub.Entry( - last_atime=i, - url='http://example.com/{}'.format(i), - title='title{}'.format(i)) - for i in range(100000)] + r = range(100000) + entries = { + 'last_atime': list(r), + 'url': ['http://example.com/{}'.format(i) for i in r], + 'title': ['title{}'.format(i) for i in r] + } web_history_stub.insert_batch(entries) - quickmark_manager_stub.marks = collections.OrderedDict( - (e.title, e.url) - for e in entries[0:1000]) + quickmark_manager_stub.marks = collections.OrderedDict([ + ('title{}'.format(i), 'example.com/{}'.format(i)) + for i in range(1000)]) - bookmark_manager_stub.marks = collections.OrderedDict( - (e.url, e.title) - for e in entries[0:1000]) + bookmark_manager_stub.marks = collections.OrderedDict([ + ('example.com/{}'.format(i), 'title{}'.format(i)) + for i in range(1000)]) def bench(): model = urlmodel.url() diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 3216400f0..afd952255 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -35,26 +35,67 @@ def test_init(): def test_insert(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) with qtbot.waitSignal(table.changed): - table.insert(name='one', val=1, lucky=False) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) with qtbot.waitSignal(table.changed): - table.insert(name='wan', val=1, lucky=False) + table.insert({'name': 'wan', 'val': 1, 'lucky': False}) -def test_insert_or_replace(qtbot): +def test_insert_replace(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], constraints={'name': 'PRIMARY KEY'}) with qtbot.waitSignal(table.changed): - table.insert_or_replace(name='one', val=1, lucky=False) + table.insert({'name': 'one', 'val': 1, 'lucky': False}, replace=True) with qtbot.waitSignal(table.changed): - table.insert_or_replace(name='one', val=11, lucky=True) + table.insert({'name': 'one', 'val': 11, 'lucky': True}, replace=True) assert list(table) == [('one', 11, True)] + with pytest.raises(sql.SqlException): + table.insert({'name': 'one', 'val': 11, 'lucky': True}, replace=False) + + +def test_insert_batch(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) + + with qtbot.waitSignal(table.changed): + table.insert_batch({'name': ['one', 'nine', 'thirteen'], + 'val': [1, 9, 13], + 'lucky': [False, False, True]}) + + assert list(table) == [('one', 1, False), + ('nine', 9, False), + ('thirteen', 13, True)] + + +def test_insert_batch_replace(qtbot): + table = sql.SqlTable('Foo', ['name', 'val', 'lucky'], + constraints={'name': 'PRIMARY KEY'}) + + with qtbot.waitSignal(table.changed): + table.insert_batch({'name': ['one', 'nine', 'thirteen'], + 'val': [1, 9, 13], + 'lucky': [False, False, True]}) + + with qtbot.waitSignal(table.changed): + table.insert_batch({'name': ['one', 'nine'], + 'val': [11, 19], + 'lucky': [True, True]}, + replace=True) + + assert list(table) == [('thirteen', 13, True), + ('one', 11, True), + ('nine', 19, True)] + + with pytest.raises(sql.SqlException): + table.insert_batch({'name': ['one', 'nine'], + 'val': [11, 19], + 'lucky': [True, True]}) + def test_iter(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(name='one', val=1, lucky=False) - table.insert(name='nine', val=9, lucky=False) - table.insert(name='thirteen', val=13, lucky=True) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) + table.insert({'name': 'nine', 'val': 9, 'lucky': False}) + table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) assert list(table) == [('one', 1, False), ('nine', 9, False), ('thirteen', 13, True)] @@ -73,15 +114,15 @@ def test_iter(): def test_select(rows, sort_by, sort_order, limit, result): table = sql.SqlTable('Foo', ['a', 'b']) for row in rows: - table.insert(**row) + table.insert(row) assert list(table.select(sort_by, sort_order, limit)) == result def test_delete(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(name='one', val=1, lucky=False) - table.insert(name='nine', val=9, lucky=False) - table.insert(name='thirteen', val=13, lucky=True) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) + table.insert({'name': 'nine', 'val': 9, 'lucky': False}) + table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) with pytest.raises(KeyError): table.delete('nope', 'name') with qtbot.waitSignal(table.changed): @@ -95,19 +136,19 @@ def test_delete(qtbot): def test_len(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) assert len(table) == 0 - table.insert(name='one', val=1, lucky=False) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) assert len(table) == 1 - table.insert(name='nine', val=9, lucky=False) + table.insert({'name': 'nine', 'val': 9, 'lucky': False}) assert len(table) == 2 - table.insert(name='thirteen', val=13, lucky=True) + table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) assert len(table) == 3 def test_contains(): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(name='one', val=1, lucky=False) - table.insert(name='nine', val=9, lucky=False) - table.insert(name='thirteen', val=13, lucky=True) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) + table.insert({'name': 'nine', 'val': 9, 'lucky': False}) + table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) name_query = table.contains_query('name') val_query = table.contains_query('val') @@ -126,9 +167,9 @@ def test_contains(): def test_delete_all(qtbot): table = sql.SqlTable('Foo', ['name', 'val', 'lucky']) - table.insert(name='one', val=1, lucky=False) - table.insert(name='nine', val=9, lucky=False) - table.insert(name='thirteen', val=13, lucky=True) + table.insert({'name': 'one', 'val': 1, 'lucky': False}) + table.insert({'name': 'nine', 'val': 9, 'lucky': False}) + table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) with qtbot.waitSignal(table.changed): table.delete_all() assert list(table) == [] From c7a18a8b8d9ff723216d4535257e22e821d67aaf Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 13 Jun 2017 21:14:47 -0400 Subject: [PATCH 118/337] Fix tests for recent sql changes --- tests/end2end/features/test_history_bdd.py | 2 +- tests/unit/completion/test_sqlcategory.py | 25 ++++++++-------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index c1000d6ce..f5639f5f3 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -30,7 +30,7 @@ def check_history(quteproc, httpbin, tmpdir, expected): path = tmpdir / 'history' quteproc.send_cmd(':debug-dump-history "{}"'.format(path)) quteproc.wait_for(category='message', loglevel=logging.INFO, - message='Dumped history to {}.'.format(path)) + message='Dumped history to {}'.format(path)) with open(path, 'r', encoding='utf-8') as f: # ignore access times, they will differ in each run diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index d81fa8cf0..ad1ff7e75 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -61,7 +61,7 @@ pytestmark = pytest.mark.usefixtures('init_sql') def test_sorting(sort_by, sort_order, data, expected): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: - table.insert(a=row[0], b=row[1], c=row[2]) + table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) cat.set_pattern('') @@ -117,7 +117,7 @@ def test_set_pattern(pattern, filter_cols, before, after): """Validate the filtering and sorting results of set_pattern.""" table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: - table.insert(a=row[0], b=row[1], c=row[2]) + table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) cat.set_pattern(pattern) @@ -126,7 +126,7 @@ def test_set_pattern(pattern, filter_cols, before, after): def test_select(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert(a='foo', b='bar', c='baz') + table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') cat.set_pattern('') utils.validate_model(cat, [('bar', 'baz', 'foo')]) @@ -134,8 +134,8 @@ def test_select(): def test_where(): table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert(a='foo', b='bar', c=False) - table.insert(a='baz', b='biz', c=True) + table.insert({'a': 'foo', 'b': 'bar', 'c': False}) + table.insert({'a': 'baz', 'b': 'biz', 'c': True}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') cat.set_pattern('') utils.validate_model(cat, [('foo', 'bar', False)]) @@ -143,18 +143,11 @@ def test_where(): def test_group(): table = sql.SqlTable('Foo', ['a', 'b']) - table.insert(a='foo', b=1) - table.insert(a='bar', b=3) - table.insert(a='foo', b=2) - table.insert(a='bar', b=0) + table.insert({'a': 'foo', 'b': 1}) + table.insert({'a': 'bar', 'b': 3}) + table.insert({'a': 'foo', 'b': 2}) + table.insert({'a': 'bar', 'b': 0}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='a, max(b)', group_by='a') cat.set_pattern('') utils.validate_model(cat, [('bar', 3), ('foo', 2)]) - - -def test_entry(): - table = sql.SqlTable('Foo', ['a', 'b', 'c']) - assert hasattr(table.Entry, 'a') - assert hasattr(table.Entry, 'b') - assert hasattr(table.Entry, 'c') From 891a6bcf1437156f95212bb3052c336e934e8147 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 14 Jun 2017 07:15:42 -0400 Subject: [PATCH 119/337] Fix flake8 errors --- qutebrowser/browser/history.py | 2 +- tests/unit/misc/test_sql.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e314f6c5e..6df09ce16 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -158,7 +158,7 @@ class WebHistory(sql.SqlTable): self.completion.insert({'url': url_str, 'title': title, 'last_atime': atime}, - replace=True) + replace=True) def _parse_entry(self, line): """Parse a history line like '12345 http://example.com title'.""" diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index afd952255..1f67f0bee 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -79,7 +79,7 @@ def test_insert_batch_replace(qtbot): table.insert_batch({'name': ['one', 'nine'], 'val': [11, 19], 'lucky': [True, True]}, - replace=True) + replace=True) assert list(table) == [('thirteen', 13, True), ('one', 11, True), From 051d2665f3ec4d83904e29fe4f5c80befef9d725 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 14 Jun 2017 07:17:30 -0400 Subject: [PATCH 120/337] Fix signal type error in CompletionView. On Travis CI we are sometimes seeing: ``` CompletionView.selection_changed[str].emit(): argument 1 has unexpected type 'int' ``` Cast the data to a string before emitting it just to be safe. --- qutebrowser/completion/completionwidget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 7ea4edf13..60ecd5b88 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -344,7 +344,7 @@ class CompletionView(QTreeView): indexes = selected.indexes() if not indexes: return - data = self.model().data(indexes[0]) + data = str(self.model().data(indexes[0])) self.selection_changed.emit(data) def resizeEvent(self, e): From 9f94f28181333c8023ea96cfd10d67e60dc70491 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 19 Jun 2017 09:14:03 +0200 Subject: [PATCH 121/337] Use :memory: for an in-memory database Using an empty string for the same purpose only started working in some recent-ish Qt/sqlite/? version, so using --version failed on Ubuntu Trusty. --- qutebrowser/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index a8fd7ad54..4eefb38d8 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -85,7 +85,7 @@ def run(args): if args.version: # we need to init sql to print the sql version # we can use an in-memory database as we just want to query the version - sql.init('') + sql.init(':memory:') print(version.version()) sys.exit(usertypes.Exit.ok) From 4296a61b9a6cc562380c2e333bd9c3d33d772f50 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 19 Jun 2017 09:19:02 +0200 Subject: [PATCH 122/337] tests: Clean up check_history --- tests/end2end/features/test_history_bdd.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index f5639f5f3..319e36aee 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -32,10 +32,9 @@ def check_history(quteproc, httpbin, tmpdir, expected): quteproc.wait_for(category='message', loglevel=logging.INFO, message='Dumped history to {}'.format(path)) - with open(path, 'r', encoding='utf-8') as f: + with path.open('r', encoding='utf-8') as f: # ignore access times, they will differ in each run - actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() - for line in f.read().splitlines()) + actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() for line in f) expected = expected.replace('(port)', str(httpbin.port)) assert actual == expected From 29ce1b381121645589cb18906682fe79b11a69b8 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 20 Jun 2017 11:56:17 +0200 Subject: [PATCH 123/337] Add column order in SqlTable._insert_query _insert_query gets called with a query and dict of values such as: {'val': 1, 'lucky': False, 'name': 'one'} Via bindValues(), we only assign a placeholder in the query string to a value, so we get a query with bindings like: INSERT INTO Foo values(:lucky,:val,:name) {':name': 'one', ':val': 1, ':lucky': False} So what we're executing is something like: INSERT INTO Foo values(false,1,"one") However, if the column order in the database doesn't happen to be the order we're passing the values in, we get the wrong values in the wrong columns. Instead, we now do: INSERT INTO Foo (lucky, val, name) values(false,1,"one") Which inserts the values in the order we intended. With Python 3.6, this just happened to work before because we always passed the keyword arguments in the table column order, and in 3.6 dicts (and thus **kwargs) happen to be ordered: https://mail.python.org/pipermail/python-dev/2016-September/146327.html --- qutebrowser/misc/sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 717ae3c0f..1f93f8c06 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -187,7 +187,8 @@ class SqlTable(QObject): def _insert_query(self, values, replace): params = ','.join(':{}'.format(key) for key in values) verb = "REPLACE" if replace else "INSERT" - return Query("{} INTO {} values({})".format(verb, self._name, params)) + return Query("{} INTO {} ({}) values({})".format( + verb, self._name, ','.join(values), params)) def insert(self, values, replace=False): """Append a row to the table. From c1776bbf9d00bb4c6bfe0c5bfc03c460e4eae411 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 20 Jun 2017 12:03:31 +0200 Subject: [PATCH 124/337] Add error message when query failed to prepare --- qutebrowser/misc/sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 1f93f8c06..3bd02352c 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -70,7 +70,8 @@ class Query(QSqlQuery): super().__init__(QSqlDatabase.database()) log.sql.debug('Preparing SQL query: "{}"'.format(querystr)) if not self.prepare(querystr): - raise SqlException('Failed to prepare query "{}"'.format(querystr)) + raise SqlException('Failed to prepare query "{}": "{}"'.format( + querystr, self.lastError().text())) self.setForwardOnly(forward_only) def __iter__(self): From da875755d112f677933e14255c986fef10cf6809 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 20 Jun 2017 12:06:17 +0200 Subject: [PATCH 125/337] Add spaces after commas in SQL queries --- qutebrowser/misc/sql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 3bd02352c..585085cc1 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -133,7 +133,7 @@ class SqlTable(QObject): column_defs = ['{} {}'.format(field, constraints.get(field, '')) for field in fields] q = Query("CREATE TABLE IF NOT EXISTS {} ({})" - .format(name, ','.join(column_defs))) + .format(name, ', '.join(column_defs))) q.run() @@ -186,10 +186,10 @@ class SqlTable(QObject): self.changed.emit() def _insert_query(self, values, replace): - params = ','.join(':{}'.format(key) for key in values) + params = ', '.join(':{}'.format(key) for key in values) verb = "REPLACE" if replace else "INSERT" return Query("{} INTO {} ({}) values({})".format( - verb, self._name, ','.join(values), params)) + verb, self._name, ', '.join(values), params)) def insert(self, values, replace=False): """Append a row to the table. From f838eb1bdc763d4f9b5b46f4778ccd56f05348d6 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 20 Jun 2017 12:11:02 +0200 Subject: [PATCH 126/337] Use named formatting for queries in sql.py --- qutebrowser/misc/sql.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 585085cc1..987fac59c 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -132,8 +132,8 @@ class SqlTable(QObject): constraints = constraints or {} column_defs = ['{} {}'.format(field, constraints.get(field, '')) for field in fields] - q = Query("CREATE TABLE IF NOT EXISTS {} ({})" - .format(name, ', '.join(column_defs))) + q = Query("CREATE TABLE IF NOT EXISTS {name} ({column_defs})" + .format(name=name, column_defs=', '.join(column_defs))) q.run() @@ -144,13 +144,13 @@ class SqlTable(QObject): name: Name of the index, should be unique. field: Name of the field to index. """ - q = Query("CREATE INDEX IF NOT EXISTS {} ON {} ({})" - .format(name, self._name, field)) + q = Query("CREATE INDEX IF NOT EXISTS {name} ON {table} ({field})" + .format(name=name, table=self._name, field=field)) q.run() def __iter__(self): """Iterate rows in the table.""" - q = Query("SELECT * FROM {}".format(self._name)) + q = Query("SELECT * FROM {table}".format(table=self._name)) q.run() return iter(q) @@ -160,12 +160,13 @@ class SqlTable(QObject): Args: field: Field to match. """ - return Query("SELECT EXISTS(SELECT * FROM {} WHERE {} = :val)" - .format(self._name, field)) + return Query( + "SELECT EXISTS(SELECT * FROM {table} WHERE {field} = :val)" + .format(table=self._name, field=field)) def __len__(self): """Return the count of rows in the table.""" - q = Query("SELECT count(*) FROM {}".format(self._name)) + q = Query("SELECT count(*) FROM {table}".format(table=self._name)) q.run() return q.value() @@ -179,7 +180,8 @@ class SqlTable(QObject): Return: The number of rows deleted. """ - q = Query("DELETE FROM {} where {} = :val".format(self._name, field)) + q = Query("DELETE FROM {table} where {field} = :val" + .format(table=self._name, field=field)) q.run(val=value) if not q.numRowsAffected(): raise KeyError('No row with {} = "{}"'.format(field, value)) @@ -188,8 +190,9 @@ class SqlTable(QObject): def _insert_query(self, values, replace): params = ', '.join(':{}'.format(key) for key in values) verb = "REPLACE" if replace else "INSERT" - return Query("{} INTO {} ({}) values({})".format( - verb, self._name, ', '.join(values), params)) + return Query("{verb} INTO {table} ({columns}) values({params})".format( + verb=verb, table=self._name, columns=', '.join(values), + params=params)) def insert(self, values, replace=False): """Append a row to the table. @@ -223,7 +226,7 @@ class SqlTable(QObject): def delete_all(self): """Remove all rows from the table.""" - Query("DELETE FROM {}".format(self._name)).run() + Query("DELETE FROM {table}".format(table=self._name)).run() self.changed.emit() def select(self, sort_by, sort_order, limit=-1): @@ -236,7 +239,8 @@ class SqlTable(QObject): Return: A prepared and executed select query. """ - q = Query('SELECT * FROM {} ORDER BY {} {} LIMIT :limit' - .format(self._name, sort_by, sort_order)) + q = Query("SELECT * FROM {table} ORDER BY {sort_by} {sort_order} " + "LIMIT :limit" + .format(table=self._name, sort_by=sort_by, sort_order=sort_order)) q.run(limit=limit) return q From 038dcff4ba5f76444a4ac6880762f51d8b190414 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 20 Jun 2017 12:16:03 +0200 Subject: [PATCH 127/337] Ignore common URL issues while importing history See #2646. This ignores the "corrupted" Apple-lookalike URLs, comments and data: URLs. --- qutebrowser/browser/history.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 6df09ce16..58961f232 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -171,10 +171,19 @@ class WebHistory(sql.SqlTable): else: raise ValueError("2 or 3 fields expected") + # http://xn--pple-43d.com/ with + # https://bugreports.qt.io/browse/QTBUG-60364 + if url in ['http://.com/', 'https://www..com/']: + return None + url = QUrl(url) if not url.isValid(): raise ValueError("Invalid URL: {}".format(url.errorString())) + # https://github.com/qutebrowser/qutebrowser/issues/2646 + if url.scheme() == 'data': + return None + # https://github.com/qutebrowser/qutebrowser/issues/670 atime = atime.lstrip('\0') @@ -223,10 +232,13 @@ class WebHistory(sql.SqlTable): completion_data = {'url': [], 'title': [], 'last_atime': []} for (i, line) in enumerate(f): line = line.strip() - if not line: + if not line or line.startswith('#'): continue try: - url, title, atime, redirect = self._parse_entry(line) + parsed = self._parse_entry(line) + if parsed is None: + continue + url, title, atime, redirect = parsed data['url'].append(url) data['title'].append(title) data['atime'].append(atime) From 6e166d139a81b53cbb50a8370d59f7269c3a10cb Mon Sep 17 00:00:00 2001 From: Marius Date: Tue, 20 Jun 2017 21:18:13 +0200 Subject: [PATCH 128/337] Fix alignment of scroll buttons in tab bar --- qutebrowser/mainwindow/tabwidget.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 2c4eb6eab..9168a28b6 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -759,6 +759,12 @@ class TabBarStyle(QCommonStyle): # style differences... rct = super().subElementRect(sr, opt, widget) return rct + elif sr == QStyle.SE_TabBarScrollLeftButton: + # We need this so the left scroll button is aligned properly. + # Otherwise, empty space will be shown after the last tab even + # though the button width is set to 0 + rct = super().subElementRect(sr, opt, widget) + return rct else: return self._style.subElementRect(sr, opt, widget) From dfedddf0bdb0d803cd8b2d32d74977228c9d58da Mon Sep 17 00:00:00 2001 From: Marius Date: Tue, 20 Jun 2017 23:55:11 +0200 Subject: [PATCH 129/337] Wrap scroll button workaround in try/except for older pyqt5 versions (5.2.1) --- qutebrowser/mainwindow/tabwidget.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 9168a28b6..2096bf8e0 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -747,6 +747,8 @@ class TabBarStyle(QCommonStyle): Return: A QRect. """ + + if sr == QStyle.SE_TabBarTabText: layouts = self._tab_layout(opt) if layouts is None: @@ -759,13 +761,20 @@ class TabBarStyle(QCommonStyle): # style differences... rct = super().subElementRect(sr, opt, widget) return rct - elif sr == QStyle.SE_TabBarScrollLeftButton: - # We need this so the left scroll button is aligned properly. - # Otherwise, empty space will be shown after the last tab even - # though the button width is set to 0 - rct = super().subElementRect(sr, opt, widget) - return rct else: + try: + # We need this so the left scroll button is aligned properly. + # Otherwise, empty space will be shown after the last tab even + # though the button width is set to 0 + + # In older PyQt-versions (5.2.1) QStyle does not have this + # attribute. + if sr == QStyle.SE_TabBarScrollLeftButton: + return super().subElementRect(sr, opt, widget) + + except AttributeError: + pass + return self._style.subElementRect(sr, opt, widget) def _tab_layout(self, opt): From f3a2b84033f0f25a298386e705839271df317157 Mon Sep 17 00:00:00 2001 From: Marius Date: Tue, 20 Jun 2017 23:58:23 +0200 Subject: [PATCH 130/337] remove space --- qutebrowser/mainwindow/tabwidget.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 2096bf8e0..84cec79c8 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -747,8 +747,6 @@ class TabBarStyle(QCommonStyle): Return: A QRect. """ - - if sr == QStyle.SE_TabBarTabText: layouts = self._tab_layout(opt) if layouts is None: From 0f585eda4f5d958eada969e75bf412601a904f1e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Jun 2017 21:41:43 -0400 Subject: [PATCH 131/337] Bring history.py back to 100% coverage. The code of debug_dump_history was tweaked to handle a possible OSException that can be thrown by open, which I noticed while trying to test it. --- qutebrowser/browser/history.py | 10 ++--- tests/unit/browser/webkit/test_history.py | 54 ++++++++++++++++++++++- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 58961f232..67d6b6994 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -270,12 +270,12 @@ class WebHistory(sql.SqlTable): .format(int(x.atime), '-r' * x.redirect, x.url, x.title) for x in self.select(sort_by='atime', sort_order='asc')) - with open(dest, 'w', encoding='utf-8') as f: - try: + try: + with open(dest, 'w', encoding='utf-8') as f: f.write('\n'.join(lines)) - except OSError as e: - raise cmdexc.CommandError('Could not write history: {}', e) - message.info("Dumped history to {}".format(dest)) + message.info("Dumped history to {}".format(dest)) + except OSError as e: + raise cmdexc.CommandError('Could not write history: {}', e) def init(parent=None): diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 04081e7c7..01178083a 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -20,12 +20,14 @@ """Tests for the global page history.""" import logging +import os import pytest from PyQt5.QtCore import QUrl from qutebrowser.browser import history from qutebrowser.utils import objreg, urlutils, usertypes +from qutebrowser.commands import cmdexc @pytest.fixture(autouse=True) @@ -144,6 +146,8 @@ def test_add_item_invalid(qtbot, hist, caplog): ('b.com', 'title', 12345, True)]), (logging.WARNING, 'a.com', '', [('a.com', 'title', 12345, False)]), (logging.WARNING, '', '', []), + (logging.WARNING, 'data:foo', '', []), + (logging.WARNING, 'a.com', 'data:foo', []), ]) def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog): with caplog.at_level(level): @@ -219,7 +223,7 @@ def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init): assert default_interface is None -def test_read(hist, data_tmpdir, monkeypatch, stubs): +def test_import_txt(hist, data_tmpdir, monkeypatch, stubs): monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) histfile = data_tmpdir / 'history' # empty line is deliberate, to test skipping empty lines @@ -242,13 +246,39 @@ def test_read(hist, data_tmpdir, monkeypatch, stubs): assert (data_tmpdir / 'history.bak').exists() +@pytest.mark.parametrize('line', [ + '', + '#12345 http://example.com/commented', + + # https://bugreports.qt.io/browse/QTBUG-60364 + '12345 http://.com/', + '12345 https://www..com/', + + # issue #2646 + '12345 data:text/html;charset=UTF-8,%3C%21DOCTYPE%20html%20PUBLIC%20%22-', +]) +def test_import_txt_skip(hist, data_tmpdir, line, monkeypatch, stubs): + """import_txt should skip certain lines silently.""" + monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) + histfile = data_tmpdir / 'history' + histfile.write(line) + + hist.import_txt() + + assert not histfile.exists() + assert not len(hist) + + @pytest.mark.parametrize('line', [ 'xyz http://example.com/bad-timestamp', '12345', 'http://example.com/no-timestamp', '68891-r-r http://example.com/double-flag', + '68891-x http://example.com/bad-flag', + '68891 http://.com', ]) -def test_read_invalid(hist, data_tmpdir, line, monkeypatch, stubs): +def test_import_txt_invalid(hist, data_tmpdir, line, monkeypatch, stubs): + """import_txt should fail on certain lines.""" monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) histfile = data_tmpdir / 'history' histfile.write(line) @@ -259,6 +289,12 @@ def test_read_invalid(hist, data_tmpdir, line, monkeypatch, stubs): assert histfile.exists() +def test_import_txt_nonexistant(hist, data_tmpdir, monkeypatch, stubs): + """import_txt should do nothing if the history file doesn't exist.""" + monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) + hist.import_txt() + + def test_debug_dump_history(hist, tmpdir): hist.add_url(QUrl('http://example.com/1'), title="Title1", atime=12345) hist.add_url(QUrl('http://example.com/2'), title="Title2", atime=12346) @@ -272,3 +308,17 @@ def test_debug_dump_history(hist, tmpdir): '12347 http://example.com/3 Title3', '12348-r http://example.com/4 Title4'] assert histfile.read() == '\n'.join(expected) + + +def test_debug_dump_history_nonexistant(hist, tmpdir): + histfile = tmpdir / 'nonexistant' / 'history' + with pytest.raises(cmdexc.CommandError): + hist.debug_dump_history(str(histfile)) + + +def test_debug_dump_history_oserror(hist, tmpdir): + histfile = tmpdir / 'history' + histfile.write('') + os.chmod(str(histfile), 0) + with pytest.raises(cmdexc.CommandError): + hist.debug_dump_history(str(histfile)) From 63cb88a0f4eb5536b89906ae0120bc472cdc76a9 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Jun 2017 22:08:23 -0400 Subject: [PATCH 132/337] Use _cat_from_index in completionmodel.data. Keep all the category lookup inside _cat_from_idx for easier refactoring if the organization ever changes. --- qutebrowser/completion/models/completionmodel.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 2d74d94e0..62148e9d3 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -75,15 +75,18 @@ class CompletionModel(QAbstractItemModel): Return: The item data, or None on an invalid index. """ - if not index.isValid() or role != Qt.DisplayRole: + if role != Qt.DisplayRole: return None - if not index.parent().isValid(): + cat = self._cat_from_idx(index) + if cat: # category header if index.column() == 0: return self._categories[index.row()].name return None # item - cat = self._categories[index.parent().row()] + cat = self._cat_from_idx(index.parent()) + if not cat: + return None idx = cat.index(index.row(), index.column()) return cat.data(idx) From b722cc1dec6ce86e8f1499879dcc4859430b09c5 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Jun 2017 22:11:53 -0400 Subject: [PATCH 133/337] Pass invalid index to [can]FetchMore. For QSqlQueryModel, the argument should always be an invalid index: http://doc.qt.io/qt-5/qsqlquerymodel.html#canFetchMore For a QStandardItemModel, it doesn't matter. Either way, passing the top-level parent index was wrong. --- qutebrowser/completion/models/completionmodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 62148e9d3..deef088cf 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -161,14 +161,14 @@ class CompletionModel(QAbstractItemModel): """Override to forward the call to the categories.""" cat = self._cat_from_idx(parent) if cat: - return cat.canFetchMore(parent) + return cat.canFetchMore(QModelIndex()) return False def fetchMore(self, parent): """Override to forward the call to the categories.""" cat = self._cat_from_idx(parent) if cat: - cat.fetchMore(parent) + cat.fetchMore(QModelIndex()) def count(self): """Return the count of non-category items.""" From 6080830a8b1cc27f5a14d68637c16e325e1fdf9c Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Jun 2017 22:25:09 -0400 Subject: [PATCH 134/337] Fix outdated docstring and pylint error. --- qutebrowser/completion/models/listcategory.py | 6 +----- qutebrowser/misc/sql.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index acdc5cafc..ce10bbb14 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -17,11 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""The base completion model for completion in the command line. - -Module attributes: - Role: An enum of user defined model roles. -""" +"""Completion category that uses a list of tuples as a data source.""" import re diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 987fac59c..52ae13e11 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -241,6 +241,7 @@ class SqlTable(QObject): """ q = Query("SELECT * FROM {table} ORDER BY {sort_by} {sort_order} " "LIMIT :limit" - .format(table=self._name, sort_by=sort_by, sort_order=sort_order)) + .format(table=self._name, sort_by=sort_by, + sort_order=sort_order)) q.run(limit=limit) return q From df6b8b7ff52d5fb5aedf911c03f319c85a1f54b9 Mon Sep 17 00:00:00 2001 From: Marius Date: Wed, 21 Jun 2017 09:03:15 +0200 Subject: [PATCH 135/337] Update tabwidget.py --- qutebrowser/mainwindow/tabwidget.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 84cec79c8..3dd0d28da 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -764,8 +764,8 @@ class TabBarStyle(QCommonStyle): # We need this so the left scroll button is aligned properly. # Otherwise, empty space will be shown after the last tab even # though the button width is set to 0 - - # In older PyQt-versions (5.2.1) QStyle does not have this + # + # In older PyQt-versions (5.2.1) QStyle does not have this # attribute. if sr == QStyle.SE_TabBarScrollLeftButton: return super().subElementRect(sr, opt, widget) From 866f4653c77d0477f3df00af518f21565e07d8b9 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 25 Jun 2017 22:14:01 -0400 Subject: [PATCH 136/337] Fix spelling existant -> existent. --- tests/unit/browser/webkit/test_history.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 01178083a..21945b537 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -289,7 +289,7 @@ def test_import_txt_invalid(hist, data_tmpdir, line, monkeypatch, stubs): assert histfile.exists() -def test_import_txt_nonexistant(hist, data_tmpdir, monkeypatch, stubs): +def test_import_txt_nonexistent(hist, data_tmpdir, monkeypatch, stubs): """import_txt should do nothing if the history file doesn't exist.""" monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) hist.import_txt() @@ -310,8 +310,8 @@ def test_debug_dump_history(hist, tmpdir): assert histfile.read() == '\n'.join(expected) -def test_debug_dump_history_nonexistant(hist, tmpdir): - histfile = tmpdir / 'nonexistant' / 'history' +def test_debug_dump_history_nonexistent(hist, tmpdir): + histfile = tmpdir / 'nonexistent' / 'history' with pytest.raises(cmdexc.CommandError): hist.debug_dump_history(str(histfile)) From a24d7f66861d0f2c7a30e55c87e70b7d63769082 Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Mon, 26 Jun 2017 12:25:03 +0200 Subject: [PATCH 137/337] Use page title for filename with :download --- qutebrowser/browser/commands.py | 3 ++- qutebrowser/browser/qtnetworkdownloads.py | 5 +++-- tests/end2end/features/downloads.feature | 7 +++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 8fead7269..f3d2473d6 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1444,7 +1444,8 @@ class CommandDispatcher: else: qnam = tab.networkaccessmanager() download_manager.get(self._current_url(), user_agent=user_agent, - qnam=qnam, target=target) + qnam=qnam, target=target, + title=self._current_title()) @cmdutils.register(instance='command-dispatcher', scope='window') def view_source(self): diff --git a/qutebrowser/browser/qtnetworkdownloads.py b/qutebrowser/browser/qtnetworkdownloads.py index 71039dc2d..6dd7c27c9 100644 --- a/qutebrowser/browser/qtnetworkdownloads.py +++ b/qutebrowser/browser/qtnetworkdownloads.py @@ -412,7 +412,7 @@ class DownloadManager(downloads.AbstractDownloadManager): mhtml.start_download_checked, tab=tab)) message.global_bridge.ask(question, blocking=False) - def get_request(self, request, *, target=None, **kwargs): + def get_request(self, request, *, target=None, title=None, **kwargs): """Start a download with a QNetworkRequest. Args: @@ -429,7 +429,8 @@ class DownloadManager(downloads.AbstractDownloadManager): QNetworkRequest.AlwaysNetwork) if request.url().scheme().lower() != 'data': - suggested_fn = urlutils.filename_from_url(request.url()) + suggested_fn = (utils.sanitize_filename(title) + ".html" if title + else urlutils.filename_from_url(request.url())) else: # We might be downloading a binary blob embedded on a page or even # generated dynamically via javascript. We try to figure out a more diff --git a/tests/end2end/features/downloads.feature b/tests/end2end/features/downloads.feature index 95b775303..ed583cb61 100644 --- a/tests/end2end/features/downloads.feature +++ b/tests/end2end/features/downloads.feature @@ -22,6 +22,13 @@ Feature: Downloading things from a website. And I wait until the download is finished Then the downloaded file download.bin should exist + Scenario: Using :download with no URL + When I set storage -> prompt-download-directory to false + And I open data/downloads/downloads.html + And I run :download + And I wait until the download is finished + Then the downloaded file Simple downloads.html should exist + Scenario: Using hints When I set storage -> prompt-download-directory to false And I open data/downloads/downloads.html From 46161c3af0189ca2616e9bff0ac626bf2ea047f3 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 20 Jun 2017 23:04:03 -0400 Subject: [PATCH 138/337] Refactor delete_cur_item. Taking the completion widget as an argument was overly complex. The process now looks like: 1. CompletionView gets deletion request 2. CompletionView passes selected index to CompletionModel 3. CompletionModel passes the row data to the owning category 4. The category runs its custom completion function. This also fixes a bug. With the switch to the hybrid (list/sql) completion model, the view was no longer updating when items were deleted. This fixes that by ensuring the correct signals are emitted. The SQL model must be refreshed by running the query. We could try using a SqlTableModel so we can call removeRows instead. The test for deleting a url fails because qmodeltester claims the length of the query model is still 3. --- qutebrowser/browser/history.py | 9 ++ qutebrowser/completion/completionwidget.py | 8 +- .../completion/models/completionmodel.py | 15 ++- qutebrowser/completion/models/listcategory.py | 16 ++- qutebrowser/completion/models/miscmodels.py | 18 +-- qutebrowser/completion/models/sqlcategory.py | 17 ++- qutebrowser/completion/models/urlmodel.py | 57 ++++----- tests/helpers/fixtures.py | 9 ++ tests/helpers/stubs.py | 29 +++++ tests/unit/completion/test_completionmodel.py | 23 +++- .../unit/completion/test_completionwidget.py | 26 ++-- tests/unit/completion/test_models.py | 120 ++++++++++-------- 12 files changed, 220 insertions(+), 127 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 67d6b6994..e9a6ce980 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -116,6 +116,15 @@ class WebHistory(sql.SqlTable): def _do_clear(self): self.delete_all() + def delete_url(self, url): + """Remove all history entries with the given url. + + Args: + url: URL string to delete. + """ + self.delete(url, 'url') + self.completion.delete(url, 'url') + @pyqtSlot(QUrl, QUrl, str) def add_from_tab(self, url, requested_url, title): """Add a new history entry as slot, called from a BrowserTab.""" diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 60ecd5b88..415968131 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -364,9 +364,7 @@ class CompletionView(QTreeView): modes=[usertypes.KeyMode.command], scope='window') def completion_item_del(self): """Delete the current completion item.""" - if not self.currentIndex().isValid(): + index = self.currentIndex() + if not index.isValid(): raise cmdexc.CommandError("No item selected!") - if self.model().delete_cur_item is None: - raise cmdexc.CommandError("Cannot delete this item.") - else: - self.model().delete_cur_item(self) + self.model().delete_cur_item(index) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index deef088cf..55b0546d9 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -22,6 +22,7 @@ from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from qutebrowser.utils import log, qtutils +from qutebrowser.commands import cmdexc class CompletionModel(QAbstractItemModel): @@ -38,13 +39,11 @@ class CompletionModel(QAbstractItemModel): _categories: The sub-categories. """ - def __init__(self, *, column_widths=(30, 70, 0), - delete_cur_item=None, parent=None): + def __init__(self, *, column_widths=(30, 70, 0), parent=None): super().__init__(parent) self.column_widths = column_widths self._categories = [] self.pattern = '' - self.delete_cur_item = delete_cur_item def _cat_from_idx(self, index): """Return the category pointed to by the given index. @@ -217,3 +216,13 @@ class CompletionModel(QAbstractItemModel): """ cat = self._cat_from_idx(index.parent()) return cat.columns_to_filter if cat else [] + + def delete_cur_item(self, index): + """Delete the row at the given index.""" + parent = index.parent() + cat = self._cat_from_idx(parent) + if not cat: + raise cmdexc.CommandError("No category selected") + self.beginRemoveRows(parent, index.row(), index.row()) + cat.delete_cur_item(cat.index(index.row(), 0)) + self.endRemoveRows() diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index ce10bbb14..c02783e66 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -21,17 +21,19 @@ import re -from PyQt5.QtCore import QSortFilterProxyModel +from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex from PyQt5.QtGui import QStandardItem, QStandardItemModel from qutebrowser.utils import qtutils +from qutebrowser.commands import cmdexc class ListCategory(QSortFilterProxyModel): """Expose a list of items as a category for the CompletionModel.""" - def __init__(self, name, items, columns_to_filter=None, parent=None): + def __init__(self, name, items, columns_to_filter=None, + delete_func=None, parent=None): super().__init__(parent) self.name = name self.srcmodel = QStandardItemModel(parent=self) @@ -41,6 +43,7 @@ class ListCategory(QSortFilterProxyModel): for item in items: self.srcmodel.appendRow([QStandardItem(x) for x in item]) self.setSourceModel(self.srcmodel) + self.delete_func = delete_func def set_pattern(self, val): """Setter for pattern. @@ -114,3 +117,12 @@ class ListCategory(QSortFilterProxyModel): return False else: return left < right + + def delete_cur_item(self, index): + """Delete the row at the given index.""" + if not self.delete_func: + raise cmdexc.CommandError("Cannot delete this item.") + data = [self.data(index.sibling(index.row(), i)) + for i in range(self.columnCount())] + self.delete_func(data) + self.removeRow(index.row(), QModelIndex()) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 62818f767..8338452c9 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -20,7 +20,7 @@ """Functions that return miscellaneous completion models.""" from qutebrowser.config import config, configdata -from qutebrowser.utils import objreg, log, qtutils +from qutebrowser.utils import objreg, log from qutebrowser.commands import cmdutils from qutebrowser.completion.models import completionmodel, listcategory @@ -96,21 +96,14 @@ def buffer(): url_column = 1 text_column = 2 - def delete_buffer(completion): + def delete_buffer(data): """Close the selected tab.""" - index = completion.currentIndex() - qtutils.ensure_valid(index) - category = index.parent() - qtutils.ensure_valid(category) - index = category.child(index.row(), idx_column) - win_id, tab_index = index.data().split('/') + win_id, tab_index = data[0].split('/') tabbed_browser = objreg.get('tabbed-browser', scope='window', window=int(win_id)) tabbed_browser.on_tab_close_requested(int(tab_index) - 1) - model = completionmodel.CompletionModel( - column_widths=(6, 40, 54), - delete_cur_item=delete_buffer) + model = completionmodel.CompletionModel(column_widths=(6, 40, 54)) for win_id in objreg.window_registry: tabbed_browser = objreg.get('tabbed-browser', scope='window', @@ -124,7 +117,8 @@ def buffer(): tab.url().toDisplayString(), tabbed_browser.page_title(idx))) cat = listcategory.ListCategory("{}".format(win_id), tabs, - columns_to_filter=[idx_column, url_column, text_column]) + columns_to_filter=[idx_column, url_column, text_column], + delete_func=delete_buffer) model.add_category(cat) return model diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 9e88b52da..9edd5f96e 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -21,10 +21,12 @@ import re +from PyQt5.QtCore import QModelIndex from PyQt5.QtSql import QSqlQueryModel from qutebrowser.misc import sql from qutebrowser.utils import debug +from qutebrowser.commands import cmdexc class SqlCategory(QSqlQueryModel): @@ -33,7 +35,7 @@ class SqlCategory(QSqlQueryModel): def __init__(self, name, *, title=None, filter_fields, sort_by=None, sort_order=None, select='*', where=None, group_by=None, - parent=None): + delete_func=None, parent=None): """Create a new completion category backed by a sql table. Args: @@ -44,6 +46,7 @@ class SqlCategory(QSqlQueryModel): where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Either 'asc' or 'desc', if sort_by is non-None + delete_func: Callback to delete a selected item. """ super().__init__(parent=parent) self.name = title or name @@ -69,6 +72,7 @@ class SqlCategory(QSqlQueryModel): col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name)) rec = col_query.run().record() self.columns_to_filter = [rec.indexOf(n) for n in filter_fields] + self.delete_func = delete_func def set_pattern(self, pattern): """Set the pattern used to filter results. @@ -86,3 +90,14 @@ class SqlCategory(QSqlQueryModel): with debug.log_time('sql', 'Running completion query'): self._query.run(pattern=pattern) self.setQuery(self._query) + + def delete_cur_item(self, index): + """Delete the row at the given index.""" + if not self.delete_func: + raise cmdexc.CommandError("Cannot delete this item.") + data = [self.data(index.sibling(index.row(), i)) + for i in range(self.columnCount())] + self.delete_func(data) + # re-run query to reload updated table + with debug.log_time('sql', 'Running completion query'): + self._query.run() diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 8cacca873..397783d6f 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -29,31 +29,25 @@ _URLCOL = 0 _TEXTCOL = 1 -def _delete_url(completion): - """Delete the selected item. +def _delete_history(data): + urlstr = data[_URLCOL] + log.completion.debug('Deleting history entry {}'.format(urlstr)) + hist = objreg.get('web-history') + hist.delete_url(urlstr) - Args: - completion: The Completion object to use. - """ - index = completion.currentIndex() - qtutils.ensure_valid(index) - category = index.parent() - index = category.child(index.row(), _URLCOL) - catname = category.data() - qtutils.ensure_valid(category) - if catname == 'Bookmarks': - urlstr = index.data() - log.completion.debug('Deleting bookmark {}'.format(urlstr)) - bookmark_manager = objreg.get('bookmark-manager') - bookmark_manager.delete(urlstr) - elif catname == 'Quickmarks': - quickmark_manager = objreg.get('quickmark-manager') - sibling = index.sibling(index.row(), _TEXTCOL) - qtutils.ensure_valid(sibling) - name = sibling.data() - log.completion.debug('Deleting quickmark {}'.format(name)) - quickmark_manager.delete(name) +def _delete_bookmark(data): + urlstr = data[_URLCOL] + log.completion.debug('Deleting bookmark {}'.format(urlstr)) + bookmark_manager = objreg.get('bookmark-manager') + bookmark_manager.delete(urlstr) + + +def _delete_quickmark(data): + name = data[_TEXTCOL] + quickmark_manager = objreg.get('quickmark-manager') + log.completion.debug('Deleting quickmark {}'.format(name)) + quickmark_manager.delete(name) def url(): @@ -61,18 +55,18 @@ def url(): Used for the `open` command. """ - model = completionmodel.CompletionModel( - column_widths=(40, 50, 10), - delete_cur_item=_delete_url) + model = completionmodel.CompletionModel(column_widths=(40, 50, 10)) quickmarks = ((url, name) for (name, url) in objreg.get('quickmark-manager').marks.items()) bookmarks = objreg.get('bookmark-manager').marks.items() - model.add_category(listcategory.ListCategory('Quickmarks', quickmarks, - columns_to_filter=[0, 1])) - model.add_category(listcategory.ListCategory('Bookmarks', bookmarks, - columns_to_filter=[0, 1])) + model.add_category(listcategory.ListCategory( + 'Quickmarks', quickmarks, columns_to_filter=[0, 1], + delete_func=_delete_quickmark)) + model.add_category(listcategory.ListCategory( + 'Bookmarks', bookmarks, columns_to_filter=[0, 1], + delete_func=_delete_bookmark)) # replace 's to avoid breaking the query timefmt = config.get('completion', 'timestamp-format').replace("'", "`") @@ -81,6 +75,7 @@ def url(): 'CompletionHistory', title='History', sort_order='desc', sort_by='last_atime', filter_fields=['url', 'title'], - select='url, title, {}'.format(select_time)) + select='url, title, {}'.format(select_time), + delete_func=_delete_history) model.add_category(hist_cat) return model diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 1a3dfd8ea..4fd6a0731 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -257,6 +257,15 @@ def bookmark_manager_stub(stubs): objreg.delete('bookmark-manager') +@pytest.fixture +def web_history_stub(init_sql, stubs): + """Fixture which provides a fake web-history object.""" + stub = stubs.WebHistoryStub() + objreg.register('web-history', stub) + yield stub + objreg.delete('web-history') + + @pytest.fixture def session_manager_stub(stubs): """Fixture which provides a fake session-manager object.""" diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 717ce4c18..a490ad5bb 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -33,6 +33,7 @@ from qutebrowser.browser import browsertab from qutebrowser.config import configexc from qutebrowser.utils import usertypes, utils from qutebrowser.mainwindow import mainwindow +from qutebrowser.misc import sql class FakeNetworkCache(QAbstractNetworkCache): @@ -522,6 +523,34 @@ class QuickmarkManagerStub(UrlMarkManagerStub): self.delete(key) +class WebHistoryStub(sql.SqlTable): + + """Stub for the web-history object.""" + + def __init__(self): + super().__init__("History", ['url', 'title', 'atime', 'redirect']) + self.completion = sql.SqlTable("CompletionHistory", + ['url', 'title', 'last_atime']) + + def add_url(self, url, title="", *, redirect=False, atime=None): + self.insert({'url': url, 'title': title, 'atime': atime, + 'redirect': redirect}) + if not redirect: + self.completion.insert({'url': url, + 'title': title, + 'last_atime': atime}) + + + def delete_url(self, url): + """Remove all history entries with the given url. + + Args: + url: URL string to delete. + """ + self.delete(url, 'url') + self.completion.delete(url, 'url') + + class HostBlockerStub: """Stub for the host-blocker object.""" diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 8f8acced2..5f4670f97 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -23,7 +23,11 @@ from unittest import mock import hypothesis from hypothesis import strategies -from qutebrowser.completion.models import completionmodel +import pytest +from PyQt5.QtCore import QModelIndex + +from qutebrowser.completion.models import completionmodel, listcategory +from qutebrowser.commands import cmdexc @hypothesis.given(strategies.lists(min_size=0, max_size=3, @@ -72,3 +76,20 @@ def test_set_pattern(pat): model.add_category(c) model.set_pattern(pat) assert all(c.set_pattern.called_with([pat]) for c in cats) + + +def test_delete_cur_item(): + func = mock.Mock() + model = completionmodel.CompletionModel() + cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=func) + model.add_category(cat) + parent = model.index(0, 0) + model.delete_cur_item(model.index(0, 0, parent)) + func.assert_called_once_with(['foo', 'bar']) + + +def test_delete_cur_item_no_cat(): + """Test completion_item_del with no selected category.""" + model = completionmodel.CompletionModel() + with pytest.raises(cmdexc.CommandError, match='No category selected'): + model.delete_cur_item(QModelIndex()) diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 4cc59deea..03f352194 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -215,30 +215,22 @@ def test_completion_show(show, rows, quick_complete, completionview, def test_completion_item_del(completionview): """Test that completion_item_del invokes delete_cur_item in the model.""" func = mock.Mock() - model = completionmodel.CompletionModel(delete_cur_item=func) - model.add_category(listcategory.ListCategory('', [('foo',)])) + model = completionmodel.CompletionModel() + cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=func) + model.add_category(cat) completionview.set_model(model) completionview.completion_item_focus('next') completionview.completion_item_del() - assert func.called + func.assert_called_once_with(['foo', 'bar']) def test_completion_item_del_no_selection(completionview): - """Test that completion_item_del with no selected index.""" + """Test that completion_item_del with an invalid index.""" func = mock.Mock() - model = completionmodel.CompletionModel(delete_cur_item=func) - model.add_category(listcategory.ListCategory('', [('foo',)])) + model = completionmodel.CompletionModel() + cat = listcategory.ListCategory('', [('foo',)], delete_func=func) + model.add_category(cat) completionview.set_model(model) with pytest.raises(cmdexc.CommandError, match='No item selected!'): completionview.completion_item_del() - assert not func.called - - -def test_completion_item_del_no_func(completionview): - """Test completion_item_del with no delete_cur_item in the model.""" - model = completionmodel.CompletionModel() - model.add_category(listcategory.ListCategory('', [('foo',)])) - completionview.set_model(model) - completionview.completion_item_focus('next') - with pytest.raises(cmdexc.CommandError, match='Cannot delete this item.'): - completionview.completion_item_del() + func.assert_not_called diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index e9e8e6366..5677eab52 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -20,6 +20,7 @@ """Tests for completion models.""" import collections +import unittest.mock from datetime import datetime import pytest @@ -28,6 +29,7 @@ from PyQt5.QtWidgets import QTreeView from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value +from qutebrowser.utils import objreg from qutebrowser.misc import sql @@ -114,23 +116,6 @@ def _patch_config_section_desc(monkeypatch, stubs, symbol): monkeypatch.setattr(symbol, section_desc) -def _mock_view_index(model, category_num, child_num, qtbot): - """Create a tree view from a model and set the current index. - - Args: - model: model to create a fake view for. - category_idx: index of the category to select. - child_idx: index of the child item under that category to select. - """ - view = QTreeView() - qtbot.add_widget(view) - view.setModel(model) - parent = model.index(category_num, 0) - child = model.index(child_num, 0, parent=parent) - view.setCurrentIndex(child) - return view - - @pytest.fixture def quickmarks(quickmark_manager_stub): """Pre-populate the quickmark-manager stub with some quickmarks.""" @@ -152,24 +137,29 @@ def bookmarks(bookmark_manager_stub): ]) return bookmark_manager_stub - @pytest.fixture -def web_history_stub(stubs, init_sql): +def history_completion_table(init_sql): return sql.SqlTable("CompletionHistory", ['url', 'title', 'last_atime']) @pytest.fixture -def web_history(web_history_stub, init_sql): +def web_history(web_history_stub): """Pre-populate the web-history database.""" - web_history_stub.insert({'url': 'http://qutebrowser.org', - 'title': 'qutebrowser', - 'last_atime': datetime(2015, 9, 5).timestamp()}) - web_history_stub.insert({'url': 'https://python.org', - 'title': 'Welcome to Python.org', - 'last_atime': datetime(2016, 3, 8).timestamp()}) - web_history_stub.insert({'url': 'https://github.com', - 'title': 'https://github.com', - 'last_atime': datetime(2016, 5, 1).timestamp()}) + web_history_stub.add_url( + url='http://qutebrowser.org', + title='qutebrowser', + atime=datetime(2015, 9, 5).timestamp() + ) + web_history_stub.add_url( + url='https://python.org', + title='Welcome to Python.org', + atime=datetime(2016, 3, 8).timestamp() + ) + web_history_stub.add_url( + url='https://github.com', + title='https://github.com', + atime=datetime(2016, 5, 1).timestamp() + ) return web_history_stub @@ -280,7 +270,6 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, Verify that: - quickmarks, bookmarks, and urls are included - entries are sorted by access time - - redirect entries are not included - only the most recent entry is included for each url """ config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} @@ -331,16 +320,15 @@ def test_url_completion_pattern(config_stub, web_history_stub, url, title, pattern, rowcount): """Test that url completion filters by url and title.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} - web_history_stub.insert({'url': url, 'title': title, 'last_atime': 0}) + web_history_stub.add_url(url, title) model = urlmodel.url() model.set_pattern(pattern) # 2, 0 is History assert model.rowCount(model.index(2, 0)) == rowcount -def test_url_completion_delete_bookmark(qtmodeltester, config_stub, - web_history, quickmarks, bookmarks, - qtbot): +def test_url_completion_delete_bookmark(qtmodeltester, config_stub, bookmarks, + web_history, quickmarks): """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() @@ -348,12 +336,18 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - # delete item (1, 1) -> (bookmarks, 'https://github.com') - view = _mock_view_index(model, 1, 1, qtbot) - model.delete_cur_item(view) + parent = model.index(1, 0) + idx = model.index(1, 0, parent) + + # sanity checks + assert model.data(parent) == "Bookmarks" + assert model.data(idx) == 'https://github.com' + assert 'https://github.com' in bookmarks.marks + + len_before = len(bookmarks.marks) + model.delete_cur_item(idx) assert 'https://github.com' not in bookmarks.marks - assert 'https://python.org' in bookmarks.marks - assert 'http://qutebrowser.org' in bookmarks.marks + assert len_before == len(bookmarks.marks) + 1 def test_url_completion_delete_quickmark(qtmodeltester, config_stub, @@ -366,28 +360,39 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - # delete item (0, 0) -> (quickmarks, 'ddg' ) - view = _mock_view_index(model, 0, 0, qtbot) - model.delete_cur_item(view) - assert 'aw' in quickmarks.marks + parent = model.index(0, 0) + idx = model.index(0, 0, parent) + + # sanity checks + assert model.data(parent) == "Quickmarks" + assert model.data(idx) == 'https://duckduckgo.com' + assert 'ddg' in quickmarks.marks + + len_before = len(quickmarks.marks) + model.delete_cur_item(idx) assert 'ddg' not in quickmarks.marks - assert 'wiki' in quickmarks.marks + assert len_before == len(quickmarks.marks) + 1 def test_url_completion_delete_history(qtmodeltester, config_stub, - web_history, quickmarks, bookmarks, - qtbot): - """Test that deleting a history entry is a noop.""" + web_history_stub, web_history, + quickmarks, bookmarks): + """Test deleting a history entry.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - hist_before = list(web_history) - view = _mock_view_index(model, 2, 0, qtbot) - model.delete_cur_item(view) - assert list(web_history) == hist_before + parent = model.index(2, 0) + idx = model.index(1, 0, parent) + + # sanity checks + assert model.data(parent) == "History" + assert model.data(idx) == 'https://python.org' + + model.delete_cur_item(idx) + assert not web_history_stub.contains('url', 'https://python.org') def test_session_completion(qtmodeltester, session_manager_stub): @@ -431,7 +436,7 @@ def test_tab_completion(qtmodeltester, fake_web_tab, app_stub, win_registry, }) -def test_tab_completion_delete(qtmodeltester, fake_web_tab, qtbot, app_stub, +def test_tab_completion_delete(qtmodeltester, fake_web_tab, app_stub, win_registry, tabbed_browser_stubs): """Verify closing a tab by deleting it from the completion widget.""" tabbed_browser_stubs[0].tabs = [ @@ -447,9 +452,14 @@ def test_tab_completion_delete(qtmodeltester, fake_web_tab, qtbot, app_stub, qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) - view = _mock_view_index(model, 0, 1, qtbot) - qtbot.add_widget(view) - model.delete_cur_item(view) + parent = model.index(0, 0) + idx = model.index(1, 0, parent) + + # sanity checks + assert model.data(parent) == "0" + assert model.data(idx) == '0/2' + + model.delete_cur_item(idx) actual = [tab.url() for tab in tabbed_browser_stubs[0].tabs] assert actual == [QUrl('https://github.com'), QUrl('https://duckduckgo.com')] From 62a849c2dbf21b988f6a9391ef1be7ffc7473e25 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 26 Jun 2017 12:41:48 -0400 Subject: [PATCH 139/337] Fix bugs introduced in test_models --- tests/unit/completion/test_models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 5677eab52..77bb89ccf 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -137,6 +137,7 @@ def bookmarks(bookmark_manager_stub): ]) return bookmark_manager_stub + @pytest.fixture def history_completion_table(init_sql): return sql.SqlTable("CompletionHistory", ['url', 'title', 'last_atime']) @@ -593,7 +594,7 @@ def test_url_completion_benchmark(benchmark, config_stub, 'title': ['title{}'.format(i) for i in r] } - web_history_stub.insert_batch(entries) + web_history_stub.completions.insert_batch(entries) quickmark_manager_stub.marks = collections.OrderedDict([ ('title{}'.format(i), 'example.com/{}'.format(i)) From 8a5b48d374ad5ba4f826b1c5dde27322cec304a5 Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Mon, 26 Jun 2017 23:21:32 +0200 Subject: [PATCH 140/337] Add suggested_fn argument to get_request --- qutebrowser/browser/commands.py | 10 +++++++--- qutebrowser/browser/qtnetworkdownloads.py | 9 +++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index f3d2473d6..fbf3fe914 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1443,9 +1443,13 @@ class CommandDispatcher: download_manager.get_mhtml(tab, target) else: qnam = tab.networkaccessmanager() - download_manager.get(self._current_url(), user_agent=user_agent, - qnam=qnam, target=target, - title=self._current_title()) + download_manager.get( + self._current_url(), + user_agent=user_agent, + qnam=qnam, + target=target, + suggested_fn=utils.sanitize_filename(tab.title() + ".html") + ) @cmdutils.register(instance='command-dispatcher', scope='window') def view_source(self): diff --git a/qutebrowser/browser/qtnetworkdownloads.py b/qutebrowser/browser/qtnetworkdownloads.py index 6dd7c27c9..1e494b1d2 100644 --- a/qutebrowser/browser/qtnetworkdownloads.py +++ b/qutebrowser/browser/qtnetworkdownloads.py @@ -412,7 +412,7 @@ class DownloadManager(downloads.AbstractDownloadManager): mhtml.start_download_checked, tab=tab)) message.global_bridge.ask(question, blocking=False) - def get_request(self, request, *, target=None, title=None, **kwargs): + def get_request(self, request, *, target=None, suggested_fn=None, **kwargs): """Start a download with a QNetworkRequest. Args: @@ -428,9 +428,10 @@ class DownloadManager(downloads.AbstractDownloadManager): request.setAttribute(QNetworkRequest.CacheLoadControlAttribute, QNetworkRequest.AlwaysNetwork) - if request.url().scheme().lower() != 'data': - suggested_fn = (utils.sanitize_filename(title) + ".html" if title - else urlutils.filename_from_url(request.url())) + if suggested_fn is not None: + pass + elif request.url().scheme().lower() != 'data': + suggested_fn = urlutils.filename_from_url(request.url()) else: # We might be downloading a binary blob embedded on a page or even # generated dynamically via javascript. We try to figure out a more From 5e2be8a44a80257525522293cc9151971f9d3b17 Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Tue, 27 Jun 2017 08:25:59 +0200 Subject: [PATCH 141/337] Fix PEP-8 issue --- qutebrowser/browser/qtnetworkdownloads.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/qtnetworkdownloads.py b/qutebrowser/browser/qtnetworkdownloads.py index 1e494b1d2..512991bbc 100644 --- a/qutebrowser/browser/qtnetworkdownloads.py +++ b/qutebrowser/browser/qtnetworkdownloads.py @@ -412,7 +412,8 @@ class DownloadManager(downloads.AbstractDownloadManager): mhtml.start_download_checked, tab=tab)) message.global_bridge.ask(question, blocking=False) - def get_request(self, request, *, target=None, suggested_fn=None, **kwargs): + def get_request(self, request, *, target=None, + suggested_fn=None, **kwargs): """Start a download with a QNetworkRequest. Args: From 0a09758be1712e8617a9a6fc5c70118eaa7634ed Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Tue, 27 Jun 2017 12:21:25 +0200 Subject: [PATCH 142/337] Add file path to download --- tests/end2end/features/downloads.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/features/downloads.feature b/tests/end2end/features/downloads.feature index ed583cb61..8a9e8c134 100644 --- a/tests/end2end/features/downloads.feature +++ b/tests/end2end/features/downloads.feature @@ -644,7 +644,7 @@ Feature: Downloading things from a website. @qtwebengine_skip: We can't get the UA from the page there Scenario: user-agent when using :download When I open user-agent - And I run :download + And I run :download --dest user-agent And I wait until the download is finished Then the downloaded file user-agent should contain Safari/ From f06880c6e2ac8036c9e39180f669c71a64178f9b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 27 Jun 2017 08:40:43 -0400 Subject: [PATCH 143/337] Fix history completion delete function. In order to update SqlQueryModel's rowCount after re-running the query, we must call setQuery again. --- qutebrowser/completion/models/sqlcategory.py | 3 ++- tests/helpers/stubs.py | 5 ++++- tests/unit/completion/test_models.py | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 9edd5f96e..e20edad3c 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -99,5 +99,6 @@ class SqlCategory(QSqlQueryModel): for i in range(self.columnCount())] self.delete_func(data) # re-run query to reload updated table - with debug.log_time('sql', 'Running completion query'): + with debug.log_time('sql', 'Re-running completion query post-delete'): self._query.run() + self.setQuery(self._query) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index a490ad5bb..6d0160f03 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -532,6 +532,10 @@ class WebHistoryStub(sql.SqlTable): self.completion = sql.SqlTable("CompletionHistory", ['url', 'title', 'last_atime']) + def __contains__(self, url): + q = self.contains_query('url') + return q.run(val=url).value() + def add_url(self, url, title="", *, redirect=False, atime=None): self.insert({'url': url, 'title': title, 'atime': atime, 'redirect': redirect}) @@ -540,7 +544,6 @@ class WebHistoryStub(sql.SqlTable): 'title': title, 'last_atime': atime}) - def delete_url(self, url): """Remove all history entries with the given url. diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 77bb89ccf..1c1a4ce63 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -392,8 +392,9 @@ def test_url_completion_delete_history(qtmodeltester, config_stub, assert model.data(parent) == "History" assert model.data(idx) == 'https://python.org' + assert 'https://python.org' in web_history_stub model.delete_cur_item(idx) - assert not web_history_stub.contains('url', 'https://python.org') + assert 'https://python.org' not in web_history_stub def test_session_completion(qtmodeltester, session_manager_stub): @@ -594,7 +595,7 @@ def test_url_completion_benchmark(benchmark, config_stub, 'title': ['title{}'.format(i) for i in r] } - web_history_stub.completions.insert_batch(entries) + web_history_stub.completion.insert_batch(entries) quickmark_manager_stub.marks = collections.OrderedDict([ ('title{}'.format(i), 'example.com/{}'.format(i)) From 6ac940fa32ded695684e998abcd25a1fbea14b15 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 27 Jun 2017 12:33:51 -0400 Subject: [PATCH 144/337] Fix pylint/coverage errors. Ensure 100% coverage for sqlcategory and history, and fix some linter errors --- qutebrowser/completion/models/sqlcategory.py | 1 - qutebrowser/completion/models/urlmodel.py | 2 +- tests/unit/browser/webkit/test_history.py | 11 +++++++++ .../unit/completion/test_completionwidget.py | 2 +- tests/unit/completion/test_models.py | 3 --- tests/unit/completion/test_sqlcategory.py | 24 +++++++++++++++++++ tests/unit/utils/test_version.py | 1 - 7 files changed, 37 insertions(+), 7 deletions(-) diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index e20edad3c..b074980b3 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -21,7 +21,6 @@ import re -from PyQt5.QtCore import QModelIndex from PyQt5.QtSql import QSqlQueryModel from qutebrowser.misc import sql diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 397783d6f..0c63f2582 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -22,7 +22,7 @@ from qutebrowser.completion.models import (completionmodel, listcategory, sqlcategory) from qutebrowser.config import config -from qutebrowser.utils import qtutils, log, objreg +from qutebrowser.utils import log, objreg _URLCOL = 0 diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/webkit/test_history.py index 21945b537..4b317acab 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/webkit/test_history.py @@ -125,6 +125,17 @@ def test_clear_force(qtbot, tmpdir, hist): assert not len(hist) +def test_delete_url(hist): + hist.add_url(QUrl('http://example.com/'), atime=0) + hist.add_url(QUrl('http://example.com/1'), atime=0) + hist.add_url(QUrl('http://example.com/2'), atime=0) + + before = set(hist) + hist.delete_url(QUrl('http://example.com/1')) + diff = before.difference(set(hist)) + assert diff == set([('http://example.com/1', '', 0, False)]) + + @pytest.mark.parametrize('url, atime, title, redirect', [ ('http://www.example.com', 12346, 'the title', False), ('http://www.example.com', 12346, 'the title', True) diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 03f352194..1625dc60a 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -233,4 +233,4 @@ def test_completion_item_del_no_selection(completionview): completionview.set_model(model) with pytest.raises(cmdexc.CommandError, match='No item selected!'): completionview.completion_item_del() - func.assert_not_called + func.assert_not_called() diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 1c1a4ce63..959224398 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -20,16 +20,13 @@ """Tests for completion models.""" import collections -import unittest.mock from datetime import datetime import pytest from PyQt5.QtCore import QUrl -from PyQt5.QtWidgets import QTreeView from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value -from qutebrowser.utils import objreg from qutebrowser.misc import sql diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index ad1ff7e75..951a7d865 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -19,11 +19,14 @@ """Test SQL-based completions.""" +import unittest.mock + import pytest from helpers import utils from qutebrowser.misc import sql from qutebrowser.completion.models import sqlcategory +from qutebrowser.commands import cmdexc pytestmark = pytest.mark.usefixtures('init_sql') @@ -151,3 +154,24 @@ def test_group(): select='a, max(b)', group_by='a') cat.set_pattern('') utils.validate_model(cat, [('bar', 3), ('foo', 2)]) + + +def test_delete_cur_item(): + table = sql.SqlTable('Foo', ['a', 'b']) + table.insert({'a': 'foo', 'b': 1}) + table.insert({'a': 'bar', 'b': 2}) + func = unittest.mock.MagicMock() + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], delete_func=func) + cat.set_pattern('') + cat.delete_cur_item(cat.index(0, 0)) + func.assert_called_with(['foo', 1]) + + +def test_delete_cur_item_no_func(): + table = sql.SqlTable('Foo', ['a', 'b']) + table.insert({'a': 'foo', 'b': 1}) + table.insert({'a': 'bar', 'b': 2}) + cat = sqlcategory.SqlCategory('Foo', filter_fields=['a']) + cat.set_pattern('') + with pytest.raises(cmdexc.CommandError, match='Cannot delete this item'): + cat.delete_cur_item(cat.index(0, 0)) diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 13dfb09d2..c0c731f88 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -811,7 +811,6 @@ def test_chromium_version_unpatched(qapp): def test_version_output(git_commit, frozen, style, with_webkit, known_distribution, stubs, monkeypatch, init_sql): """Test version.version().""" - # pylint: disable=too-many-locals class FakeWebEngineProfile: def httpUserAgent(self): return 'Toaster/4.0.4 Chrome/CHROMIUMVERSION Teapot/4.1.8' From 4d1dbe11e89ab3cffd5477a5edfdfed3db8b7650 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Tue, 27 Jun 2017 19:02:41 -0700 Subject: [PATCH 145/337] Prompt when closing a pinned tab via the mouse Closes #2761 --- qutebrowser/browser/commands.py | 7 ++++--- qutebrowser/mainwindow/tabbedbrowser.py | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index e0b89c693..54a5213fa 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -227,7 +227,8 @@ class CommandDispatcher: self._tabbed_browser.close_tab(tab) tabbar.setSelectionBehaviorOnRemove(old_selection_behavior) - def _tab_close_prompt_if_pinned(self, tab, force, yes_action): + @staticmethod + def tab_close_prompt_if_pinned(tab, force, yes_action): """Helper method for tab_close. If tab is pinned, prompt. If everything is good, run yes_action. @@ -260,7 +261,7 @@ class CommandDispatcher: close = functools.partial(self._tab_close, tab, prev, next_, opposite) - self._tab_close_prompt_if_pinned(tab, force, close) + CommandDispatcher.tab_close_prompt_if_pinned(tab, force, close) @cmdutils.register(instance='command-dispatcher', scope='window', name='tab-pin') @@ -920,7 +921,7 @@ class CommandDispatcher: if not force: for i, tab in enumerate(self._tabbed_browser.widgets()): if _to_close(i) and tab.data.pinned: - self._tab_close_prompt_if_pinned( + CommandDispatcher.tab_close_prompt_if_pinned( tab, force, lambda: self.tab_only( prev=prev, next_=next_, force=True)) diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index e0468330e..5fb9a02f8 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -26,6 +26,7 @@ from PyQt5.QtWidgets import QSizePolicy from PyQt5.QtCore import pyqtSignal, pyqtSlot, QTimer, QUrl, QSize from PyQt5.QtGui import QIcon +from qutebrowser.browser.commands import CommandDispatcher from qutebrowser.config import config from qutebrowser.keyinput import modeman from qutebrowser.mainwindow import tabwidget @@ -366,7 +367,8 @@ class TabbedBrowser(tabwidget.TabWidget): log.webview.debug("Got invalid tab {} for index {}!".format( tab, idx)) return - self.close_tab(tab) + CommandDispatcher.tab_close_prompt_if_pinned( + tab, False, lambda: self.close_tab(tab)) @pyqtSlot(browsertab.AbstractTab) def on_window_close_requested(self, widget): From 302961a86aa0ba2a2998e8c21e0319f4f4a91f96 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Wed, 28 Jun 2017 21:15:49 -0700 Subject: [PATCH 146/337] Refactor set_tab_pinned to take a tab widget. See #2759 --- qutebrowser/browser/commands.py | 9 +++++++-- qutebrowser/mainwindow/tabbedbrowser.py | 2 +- qutebrowser/mainwindow/tabwidget.py | 9 +++++---- qutebrowser/misc/sessions.py | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 3d8516fe2..54ee0d53e 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -282,7 +282,12 @@ class CommandDispatcher: to_pin = not tab.data.pinned tab_index = self._current_index() if count is None else count - 1 cmdutils.check_overflow(tab_index + 1, 'int') - self._tabbed_browser.set_tab_pinned(tab_index, to_pin) + tab = self._cntwidget(count) + + if tab is None: + raise cmdexc.CommandError("Unable to find tab '{}'.".format(count)) + + self._tabbed_browser.set_tab_pinned(tab, to_pin) @cmdutils.register(instance='command-dispatcher', name='open', maxsplit=0, scope='window') @@ -515,7 +520,7 @@ class CommandDispatcher: newtab.data.keep_icon = True newtab.history.deserialize(history) newtab.zoom.set_factor(curtab.zoom.factor()) - new_tabbed_browser.set_tab_pinned(idx, curtab.data.pinned) + new_tabbed_browser.set_tab_pinned(newtab, curtab.data.pinned) return newtab @cmdutils.register(instance='command-dispatcher', scope='window') diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index e0468330e..94508ab63 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -342,7 +342,7 @@ class TabbedBrowser(tabwidget.TabWidget): newtab = self.tabopen(url, background=False, idx=idx) newtab.history.deserialize(history_data) - self.set_tab_pinned(idx, pinned) + self.set_tab_pinned(newtab, pinned) @pyqtSlot('QUrl', bool) def openurl(self, url, newtab): diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 2c4eb6eab..c7823f84e 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -26,7 +26,7 @@ from PyQt5.QtCore import (pyqtSignal, pyqtSlot, Qt, QSize, QRect, QPoint, QTimer, QUrl) from PyQt5.QtWidgets import (QTabWidget, QTabBar, QSizePolicy, QCommonStyle, QStyle, QStylePainter, QStyleOptionTab, - QStyleFactory) + QStyleFactory, QWidget) from PyQt5.QtGui import QIcon, QPalette, QColor from qutebrowser.utils import qtutils, objreg, utils, usertypes, log @@ -94,17 +94,18 @@ class TabWidget(QTabWidget): bar.set_tab_data(idx, 'indicator-color', color) bar.update(bar.tabRect(idx)) - def set_tab_pinned(self, idx, pinned, *, loading=False): + def set_tab_pinned(self, tab: QWidget, + pinned: bool, *, loading: bool = False) -> None: """Set the tab status as pinned. Args: - idx: The tab index. + tab: The tab to pin pinned: Pinned tab state to set. loading: Whether to ignore current data state when counting pinned_count. """ bar = self.tabBar() - tab = self.widget(idx) + idx = self.indexOf(tab) # Only modify pinned_count if we had a change # always modify pinned_count if we are loading diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 5ce4ee66d..26656e1ee 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -406,7 +406,7 @@ class SessionManager(QObject): tab_to_focus = i if new_tab.data.pinned: tabbed_browser.set_tab_pinned( - i, new_tab.data.pinned, loading=True) + new_tab, new_tab.data.pinned, loading=True) if tab_to_focus is not None: tabbed_browser.setCurrentIndex(tab_to_focus) if win.get('active', False): From 9c0c1745343f345664efa2e02b12c9da3818039d Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 12:44:02 -0400 Subject: [PATCH 147/337] Use builtin SortFilter regex functionality. With the new completion API, we no longer need a custom filterAcceptsRow function. This was necessary to handle the tree structure of the model, but now we use a separate QSortFilterProxyModel for each category, so the data it filters is flat. We can simplify the code by using the builtin setFilterRegExp. This changes the behavior a little, as now all list categories filter on all columns. This should be beneficial if anything. For example, help topics are now filtered on description in addition to name. This also seems to slightly speed up filtering, according to the url model benchmark. Before: ----------------------------------------------- benchmark: 1 tests ---------------------------------------------- Name (time in s) Min Max Mean StdDev Median IQR Outliers(*) Rounds Iterations ----------------------------------------------------------------------------------------------------------------- test_url_completion_benchmark 1.2806 1.3817 1.3195 0.0390 1.3068 0.0487 1;0 5 1 ----------------------------------------------------------------------------------------------------------------- After: ----------------------------------------------- benchmark: 1 tests ---------------------------------------------- Name (time in s) Min Max Mean StdDev Median IQR Outliers(*) Rounds Iterations ----------------------------------------------------------------------------------------------------------------- test_url_completion_benchmark 1.1183 1.1508 1.1281 0.0132 1.1241 0.0142 1;0 5 1 ----------------------------------------------------------------------------------------------------------------- --- qutebrowser/completion/models/listcategory.py | 40 ++++--------------- qutebrowser/completion/models/miscmodels.py | 4 -- qutebrowser/completion/models/urlmodel.py | 6 +-- 3 files changed, 9 insertions(+), 41 deletions(-) diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index c02783e66..ee9a5e53e 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -21,7 +21,7 @@ import re -from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex +from PyQt5.QtCore import Qt, QSortFilterProxyModel, QModelIndex, QRegExp from PyQt5.QtGui import QStandardItem, QStandardItemModel from qutebrowser.utils import qtutils @@ -32,14 +32,14 @@ class ListCategory(QSortFilterProxyModel): """Expose a list of items as a category for the CompletionModel.""" - def __init__(self, name, items, columns_to_filter=None, - delete_func=None, parent=None): + def __init__(self, name, items, delete_func=None, parent=None): super().__init__(parent) self.name = name self.srcmodel = QStandardItemModel(parent=self) self.pattern = '' - self.pattern_re = None - self.columns_to_filter = columns_to_filter or [0] + # ListCategory filters all columns + self.columns_to_filter = [0, 1, 2] + self.setFilterKeyColumn(-1) for item in items: self.srcmodel.appendRow([QStandardItem(x) for x in item]) self.setSourceModel(self.srcmodel) @@ -55,38 +55,12 @@ class ListCategory(QSortFilterProxyModel): val = re.sub(r' +', r' ', val) # See #1919 val = re.escape(val) val = val.replace(r'\ ', '.*') - self.pattern_re = re.compile(val, re.IGNORECASE) + rx = QRegExp(val, Qt.CaseInsensitive) + self.setFilterRegExp(rx) self.invalidate() sortcol = 0 self.sort(sortcol) - def filterAcceptsRow(self, row, parent): - """Custom filter implementation. - - Override QSortFilterProxyModel::filterAcceptsRow. - - Args: - row: The row of the item. - parent: The parent item QModelIndex. - - Return: - True if self.pattern is contained in item. - """ - if not self.pattern: - return True - - for col in self.columns_to_filter: - idx = self.srcmodel.index(row, col, parent) - if not idx.isValid(): # pragma: no cover - # this is a sanity check not hit by any test case - continue - data = self.srcmodel.data(idx) - if not data: - continue - elif self.pattern_re.search(data): - return True - return False - def lessThan(self, lindex, rindex): """Custom sorting implementation. diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 8338452c9..0af6060e7 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -92,9 +92,6 @@ def buffer(): Used for switching the buffer command. """ - idx_column = 0 - url_column = 1 - text_column = 2 def delete_buffer(data): """Close the selected tab.""" @@ -117,7 +114,6 @@ def buffer(): tab.url().toDisplayString(), tabbed_browser.page_title(idx))) cat = listcategory.ListCategory("{}".format(win_id), tabs, - columns_to_filter=[idx_column, url_column, text_column], delete_func=delete_buffer) model.add_category(cat) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 0c63f2582..4f5fdeae9 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -62,11 +62,9 @@ def url(): bookmarks = objreg.get('bookmark-manager').marks.items() model.add_category(listcategory.ListCategory( - 'Quickmarks', quickmarks, columns_to_filter=[0, 1], - delete_func=_delete_quickmark)) + 'Quickmarks', quickmarks, delete_func=_delete_quickmark)) model.add_category(listcategory.ListCategory( - 'Bookmarks', bookmarks, columns_to_filter=[0, 1], - delete_func=_delete_bookmark)) + 'Bookmarks', bookmarks, delete_func=_delete_bookmark)) # replace 's to avoid breaking the query timefmt = config.get('completion', 'timestamp-format').replace("'", "`") From c007f592b3867d318ccfdd8656b982cb2a317693 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 20:43:42 -0400 Subject: [PATCH 148/337] Use more intuitive argument order in sql.delete. --- qutebrowser/browser/history.py | 4 ++-- qutebrowser/completion/models/sqlcategory.py | 1 - qutebrowser/misc/sql.py | 4 ++-- tests/unit/misc/test_sql.py | 6 +++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e9a6ce980..86c6a6cd9 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -122,8 +122,8 @@ class WebHistory(sql.SqlTable): Args: url: URL string to delete. """ - self.delete(url, 'url') - self.completion.delete(url, 'url') + self.delete('url', url) + self.completion.delete('url', url) @pyqtSlot(QUrl, QUrl, str) def add_from_tab(self, url, requested_url, title): diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index b074980b3..001a844b0 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -78,7 +78,6 @@ class SqlCategory(QSqlQueryModel): Args: pattern: string pattern to filter by. - columns_to_filter: indices of columns to apply pattern to. """ # escape to treat a user input % or _ as a literal, not a wildcard pattern = pattern.replace('%', '\\%') diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 52ae13e11..4db3fb899 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -170,12 +170,12 @@ class SqlTable(QObject): q.run() return q.value() - def delete(self, value, field): + def delete(self, field, value): """Remove all rows for which `field` equals `value`. Args: - value: Key value to delete. field: Field to use as the key. + value: Key value to delete. Return: The number of rows deleted. diff --git a/tests/unit/misc/test_sql.py b/tests/unit/misc/test_sql.py index 1f67f0bee..8997afc3b 100644 --- a/tests/unit/misc/test_sql.py +++ b/tests/unit/misc/test_sql.py @@ -124,12 +124,12 @@ def test_delete(qtbot): table.insert({'name': 'nine', 'val': 9, 'lucky': False}) table.insert({'name': 'thirteen', 'val': 13, 'lucky': True}) with pytest.raises(KeyError): - table.delete('nope', 'name') + table.delete('name', 'nope') with qtbot.waitSignal(table.changed): - table.delete('thirteen', 'name') + table.delete('name', 'thirteen') assert list(table) == [('one', 1, False), ('nine', 9, False)] with qtbot.waitSignal(table.changed): - table.delete(False, field='lucky') + table.delete('lucky', False) assert not list(table) From 262b028ee9a13ad9b8afa8f1a9f989da497856e4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 20:49:05 -0400 Subject: [PATCH 149/337] Match error message in lineparser test. --- tests/unit/misc/test_lineparser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/misc/test_lineparser.py b/tests/unit/misc/test_lineparser.py index dcdb724cd..cd4a64274 100644 --- a/tests/unit/misc/test_lineparser.py +++ b/tests/unit/misc/test_lineparser.py @@ -114,7 +114,8 @@ class TestLineParser: def test_double_open(self, lineparser): """Test if save() bails on an already open file.""" with lineparser._open('r'): - with pytest.raises(IOError): + with pytest.raises(IOError, + match="Refusing to double-open LineParser."): lineparser.save() def test_prepare_save(self, tmpdir, lineparser): From fd07c571e5ea77beb7c19ecf593cb26a9913ac5b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 20:54:56 -0400 Subject: [PATCH 150/337] Store pattern in completion view, not model. The pattern property is used for highlighting. It is purely display-related, so it should be in the view rather than the model. --- qutebrowser/completion/completiondelegate.py | 3 ++- qutebrowser/completion/completionwidget.py | 3 +++ qutebrowser/completion/models/completionmodel.py | 4 ---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/qutebrowser/completion/completiondelegate.py b/qutebrowser/completion/completiondelegate.py index 5813d6dcb..b2a933cef 100644 --- a/qutebrowser/completion/completiondelegate.py +++ b/qutebrowser/completion/completiondelegate.py @@ -196,7 +196,8 @@ class CompletionItemDelegate(QStyledItemDelegate): self._doc.setDocumentMargin(2) if index.parent().isValid(): - pattern = index.model().pattern + view = self.parent() + pattern = view.pattern columns_to_filter = index.model().columns_to_filter(index) if index.column() in columns_to_filter and pattern: repl = r'\g<0>' diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 415968131..9b8d31ad6 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -40,6 +40,7 @@ class CompletionView(QTreeView): headers, and children show as flat list. Attributes: + pattern: Current filter pattern, used for highlighting. _win_id: The ID of the window this CompletionView is associated with. _height: The height to use for the CompletionView. _height_perc: Either None or a percentage if height should be relative. @@ -106,6 +107,7 @@ class CompletionView(QTreeView): def __init__(self, win_id, parent=None): super().__init__(parent) + self.pattern = '' self._win_id = win_id # FIXME handle new aliases. # objreg.get('config').changed.connect(self.init_command_completion) @@ -288,6 +290,7 @@ class CompletionView(QTreeView): self.expand(model.index(i, 0)) def set_pattern(self, pattern): + self.pattern = pattern with debug.log_time(log.completion, 'Set pattern {}'.format(pattern)): self.model().set_pattern(pattern) self._resize_columns() diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 55b0546d9..9eb57fe88 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -35,7 +35,6 @@ class CompletionModel(QAbstractItemModel): Attributes: column_widths: The width percentages of the columns used in the completion view. - pattern: Current filter pattern, used for highlighting. _categories: The sub-categories. """ @@ -43,7 +42,6 @@ class CompletionModel(QAbstractItemModel): super().__init__(parent) self.column_widths = column_widths self._categories = [] - self.pattern = '' def _cat_from_idx(self, index): """Return the category pointed to by the given index. @@ -180,8 +178,6 @@ class CompletionModel(QAbstractItemModel): pattern: The filter pattern to set. """ log.completion.debug("Setting completion pattern '{}'".format(pattern)) - # TODO: should pattern be saved in the view layer instead? - self.pattern = pattern for cat in self._categories: cat.set_pattern(pattern) From 1e489325c4129e3c6a51005bddaea3dd7f68bca3 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 20:58:15 -0400 Subject: [PATCH 151/337] Assert if index is invalid in delete_cur_item. CompletionView already checks the index, so an error here shouldn't happen. --- qutebrowser/completion/models/completionmodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 9eb57fe88..a97af2342 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -215,10 +215,10 @@ class CompletionModel(QAbstractItemModel): def delete_cur_item(self, index): """Delete the row at the given index.""" + qtutils.ensure_valid(index) parent = index.parent() cat = self._cat_from_idx(parent) - if not cat: - raise cmdexc.CommandError("No category selected") + assert cat, "CompletionView sent invalid index for deletion" self.beginRemoveRows(parent, index.row(), index.row()) cat.delete_cur_item(cat.index(index.row(), 0)) self.endRemoveRows() From c1f5e77fc69a3c23000416f7c9907dd2bf3e958d Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 21:44:44 -0400 Subject: [PATCH 152/337] Implement "Current" completion for bind. When binding a key, the first row will be the current binding if the key is already bound. This should make it easier for users to tell when they are binding a key that is already bound, and what it is bound to. --- qutebrowser/completion/models/miscmodels.py | 12 +++++++++--- tests/unit/completion/test_models.py | 3 +++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 0af6060e7..00fabb031 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -120,14 +120,20 @@ def buffer(): return model -def bind(_key): +def bind(key): """A CompletionModel filled with all bindable commands and descriptions. Args: - _key: the key being bound. + key: the key being bound. """ - # TODO: offer a 'Current binding' completion based on the key. model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) + cmd_name = objreg.get('key-config').get_bindings_for('normal').get(key) + + if cmd_name: + cmd = cmdutils.cmd_dict.get(cmd_name) + data = [(cmd_name, cmd.desc, key)] + model.add_category(listcategory.ListCategory("Current", data)) + cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) model.add_category(listcategory.ListCategory("Commands", cmdlist)) return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 959224398..b7b688a81 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -567,6 +567,9 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, qtmodeltester.check(model) _check_completions(model, { + "Current": [ + ('stop', 'stop qutebrowser', 's'), + ], "Commands": [ ('drop', 'drop all user data', ''), ('hide', '', ''), From 22880926b1aa1dd1ee97b3a0d49d652223e814f0 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 29 Jun 2017 21:46:09 -0400 Subject: [PATCH 153/337] Fix WebHistoryStub for delete argument change --- tests/helpers/stubs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 6d0160f03..05d91fac0 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -550,8 +550,8 @@ class WebHistoryStub(sql.SqlTable): Args: url: URL string to delete. """ - self.delete(url, 'url') - self.completion.delete(url, 'url') + self.delete('url', url) + self.completion.delete('url', url) class HostBlockerStub: From 596dee69d61baeb5cdbed8d9d044597f2038d275 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Thu, 29 Jun 2017 20:04:02 -0700 Subject: [PATCH 154/337] Clean up pin_tab Also add a test case for :pin-tab with an invalid count --- qutebrowser/browser/commands.py | 9 +-------- tests/end2end/features/tabs.feature | 11 +++++++++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 54ee0d53e..f21115db5 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -277,16 +277,9 @@ class CommandDispatcher: """ tab = self._cntwidget(count) if tab is None: - return + raise cmdexc.CommandError("Tab {} does not exist!".format(count)) to_pin = not tab.data.pinned - tab_index = self._current_index() if count is None else count - 1 - cmdutils.check_overflow(tab_index + 1, 'int') - tab = self._cntwidget(count) - - if tab is None: - raise cmdexc.CommandError("Unable to find tab '{}'.".format(count)) - self._tabbed_browser.set_tab_pinned(tab, to_pin) @cmdutils.register(instance='command-dispatcher', name='open', diff --git a/tests/end2end/features/tabs.feature b/tests/end2end/features/tabs.feature index 9980b448f..1d3f22b10 100644 --- a/tests/end2end/features/tabs.feature +++ b/tests/end2end/features/tabs.feature @@ -1073,6 +1073,17 @@ Feature: Tab management - data/numbers/2.txt (pinned) - data/numbers/3.txt (active) + Scenario: :tab-pin with an invalid count + When I open data/numbers/1.txt + And I open data/numbers/2.txt in a new tab + And I open data/numbers/3.txt in a new tab + And I run :tab-pin with count 23 + Then the error "Tab 23 does not exist!" should be shown + And the following tabs should be open: + - data/numbers/1.txt + - data/numbers/2.txt + - data/numbers/3.txt (active) + Scenario: Pinned :tab-close prompt yes When I open data/numbers/1.txt And I run :tab-pin From 2fbadc46d21e5a76d94904e951105464c1acebfb Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Fri, 30 Jun 2017 09:57:39 -0700 Subject: [PATCH 155/337] Remove error when count is invalid to :tab-pin --- qutebrowser/browser/commands.py | 2 +- tests/end2end/features/tabs.feature | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index f21115db5..0b448a845 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -277,7 +277,7 @@ class CommandDispatcher: """ tab = self._cntwidget(count) if tab is None: - raise cmdexc.CommandError("Tab {} does not exist!".format(count)) + return to_pin = not tab.data.pinned self._tabbed_browser.set_tab_pinned(tab, to_pin) diff --git a/tests/end2end/features/tabs.feature b/tests/end2end/features/tabs.feature index 1d3f22b10..8097f390e 100644 --- a/tests/end2end/features/tabs.feature +++ b/tests/end2end/features/tabs.feature @@ -1078,8 +1078,7 @@ Feature: Tab management And I open data/numbers/2.txt in a new tab And I open data/numbers/3.txt in a new tab And I run :tab-pin with count 23 - Then the error "Tab 23 does not exist!" should be shown - And the following tabs should be open: + Then the following tabs should be open: - data/numbers/1.txt - data/numbers/2.txt - data/numbers/3.txt (active) From 3b53ec1cb6c136ffb12e46f3b532e546cb0bd421 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 3 Jul 2017 09:40:39 +0200 Subject: [PATCH 156/337] Skip tests with permission changes if they didn't work This e.g. wouldn't work inside of a Docker container otherwise. --- tests/end2end/features/downloads.feature | 4 ++-- tests/end2end/features/test_downloads_bdd.py | 9 +++++++++ tests/unit/misc/test_editor.py | 10 ++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/end2end/features/downloads.feature b/tests/end2end/features/downloads.feature index 95b775303..07e062610 100644 --- a/tests/end2end/features/downloads.feature +++ b/tests/end2end/features/downloads.feature @@ -579,9 +579,9 @@ Feature: Downloading things from a website. And I wait until the download is finished Then the downloaded file content-size should exist - @posix Scenario: Downloading to unwritable destination - When I set storage -> prompt-download-directory to false + When the unwritable dir is unwritable + And I set storage -> prompt-download-directory to false And I run :download http://localhost:(port)/data/downloads/download.bin --dest (tmpdir)/downloads/unwritable Then the error "Download error: Permission denied" should be shown diff --git a/tests/end2end/features/test_downloads_bdd.py b/tests/end2end/features/test_downloads_bdd.py index 25eb52aad..4be175a66 100644 --- a/tests/end2end/features/test_downloads_bdd.py +++ b/tests/end2end/features/test_downloads_bdd.py @@ -21,6 +21,7 @@ import os import sys import shlex +import pytest import pytest_bdd as bdd bdd.scenarios('downloads.feature') @@ -53,6 +54,14 @@ def clean_old_downloads(quteproc): quteproc.send_cmd(':download-clear') +@bdd.when("the unwritable dir is unwritable") +def check_unwritable(tmpdir): + unwritable = tmpdir / 'downloads' / 'unwritable' + if os.access(str(unwritable), os.W_OK): + # Docker container or similar + pytest.skip("Unwritable dir was writable") + + @bdd.when("I wait until the download is finished") def wait_for_download_finished(quteproc): quteproc.wait_for(category='downloads', message='Download * finished') diff --git a/tests/unit/misc/test_editor.py b/tests/unit/misc/test_editor.py index de9125c8b..70886cda6 100644 --- a/tests/unit/misc/test_editor.py +++ b/tests/unit/misc/test_editor.py @@ -123,24 +123,30 @@ class TestFileHandling: os.remove(filename) - @pytest.mark.posix def test_unreadable(self, message_mock, editor, caplog): """Test file handling when closing with an unreadable file.""" editor.edit("") filename = editor._file.name assert os.path.exists(filename) os.chmod(filename, 0o077) + if os.access(filename, os.R_OK): + # Docker container or similar + pytest.skip("File was still readable") + with caplog.at_level(logging.ERROR): editor._proc.finished.emit(0, QProcess.NormalExit) assert not os.path.exists(filename) msg = message_mock.getmsg(usertypes.MessageLevel.error) assert msg.text.startswith("Failed to read back edited file: ") - @pytest.mark.posix def test_unwritable(self, monkeypatch, message_mock, editor, tmpdir, caplog): """Test file handling when the initial file is not writable.""" tmpdir.chmod(0) + if os.access(str(tmpdir), os.W_OK): + # Docker container or similar + pytest.skip("File was still writable") + monkeypatch.setattr(editormod.tempfile, 'tempdir', str(tmpdir)) with caplog.at_level(logging.ERROR): From 629038632c6cc8c0ecd8b6a7a32d59ef4d5b9d02 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 3 Jul 2017 09:42:02 +0200 Subject: [PATCH 157/337] Add proxy support for QtWebEngine and Qt 5.7.1 This used to give us crashes in libproxy: https://github.com/libproxy/libproxy/issues/45 https://bugreports.qt.io/browse/QTBUG-56852 However, trying again with Qt 5.7.1 on Debian and from PyPI, this doesn't happen anymore, so it was probably something with how Archlinux handled things. See #2082, #2775. Reverts fd29528e4f0c3ef445a85ea1d9e2c96bcc3e37ed --- CHANGELOG.asciidoc | 1 + doc/help/settings.asciidoc | 2 -- qutebrowser/app.py | 6 ++---- qutebrowser/browser/webkit/webkittab.py | 7 ------- qutebrowser/config/configdata.py | 8 ++------ scripts/dev/src2asciidoc.py | 3 +-- 6 files changed, 6 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 406400413..ea7481030 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -45,6 +45,7 @@ Added customize statusbar colors for private windows. - New `{private}` field displaying `[Private Mode]` for `ui -> window-title-format` and `tabs -> title-format`. +- (QtWebEngine) Proxy support with Qt 5.7.1 too Changed ~~~~~~~ diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 9c362dbca..b038d462d 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -789,8 +789,6 @@ The proxy to use. In addition to the listed values, you can use a `socks://...` or `http://...` URL. -This setting only works with Qt 5.8 or newer when using the QtWebEngine backend. - Valid values: * +system+: Use the system wide proxy. diff --git a/qutebrowser/app.py b/qutebrowser/app.py index a3a855f9a..65e8d8181 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -410,10 +410,8 @@ def _init_modules(args, crash_handler): log.init.debug("Initializing network...") networkmanager.init() - if qtutils.version_check('5.8'): - # Otherwise we can only initialize it for QtWebKit because of crashes - log.init.debug("Initializing proxy...") - proxy.init() + log.init.debug("Initializing proxy...") + proxy.init() log.init.debug("Initializing readline-bridge...") readline_bridge = readline.ReadlineBridge() diff --git a/qutebrowser/browser/webkit/webkittab.py b/qutebrowser/browser/webkit/webkittab.py index fa626e863..c29aa15cb 100644 --- a/qutebrowser/browser/webkit/webkittab.py +++ b/qutebrowser/browser/webkit/webkittab.py @@ -33,7 +33,6 @@ from PyQt5.QtWebKit import QWebSettings from PyQt5.QtPrintSupport import QPrinter from qutebrowser.browser import browsertab -from qutebrowser.browser.network import proxy from qutebrowser.browser.webkit import webview, tabhistory, webkitelem from qutebrowser.browser.webkit.network import webkitqutescheme from qutebrowser.utils import qtutils, objreg, usertypes, utils, log, debug @@ -42,12 +41,6 @@ from qutebrowser.utils import qtutils, objreg, usertypes, utils, log, debug def init(): """Initialize QtWebKit-specific modules.""" qapp = QApplication.instance() - - if not qtutils.version_check('5.8'): - # Otherwise we initialize it globally in app.py - log.init.debug("Initializing proxy...") - proxy.init() - log.init.debug("Initializing js-bridge...") js_bridge = webkitqutescheme.JSBridge(qapp) objreg.register('js-bridge', js_bridge) diff --git a/qutebrowser/config/configdata.py b/qutebrowser/config/configdata.py index 28cf41ac5..32d80f3b7 100644 --- a/qutebrowser/config/configdata.py +++ b/qutebrowser/config/configdata.py @@ -437,14 +437,10 @@ def data(readonly=False): "User agent to send. Empty to send the default."), ('proxy', - SettingValue(typ.Proxy(), 'system', - backends=(None if qtutils.version_check('5.8') - else [usertypes.Backend.QtWebKit])), + SettingValue(typ.Proxy(), 'system'), "The proxy to use.\n\n" "In addition to the listed values, you can use a `socks://...` " - "or `http://...` URL.\n\n" - "This setting only works with Qt 5.8 or newer when using the " - "QtWebEngine backend."), + "or `http://...` URL."), ('proxy-dns-requests', SettingValue(typ.Bool(), 'true', diff --git a/scripts/dev/src2asciidoc.py b/scripts/dev/src2asciidoc.py index 7445a99c6..a202edeb3 100755 --- a/scripts/dev/src2asciidoc.py +++ b/scripts/dev/src2asciidoc.py @@ -365,8 +365,7 @@ def generate_commands(filename): def _generate_setting_section(f, sectname, sect): """Generate documentation for a single section.""" - version_dependent_options = [('network', 'proxy'), - ('general', 'print-element-backgrounds')] + version_dependent_options = [('general', 'print-element-backgrounds')] for optname, option in sect.items(): f.write("\n") f.write('[[{}-{}]]'.format(sectname, optname) + "\n") From f2dbff92f49a39a214f89d6f5815b03f921e66a1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 2 Jul 2017 12:53:54 -0400 Subject: [PATCH 158/337] Check for PyQt.QtSql and sqlite in earlyinit. Show a graphical error box with install instructions if PyQt.QtSql is not found, rather than failing with CLI errors. Also show an error box if the sqlite driver is not available. --- qutebrowser/misc/earlyinit.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index 9391288de..95f61708a 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -336,6 +336,13 @@ def check_libraries(backend): "http://pyyaml.org/download/pyyaml/ (py3.4) " "or Install via pip.", pip="PyYAML"), + 'PyQt5.QtSql': + _missing_str("PyQt5.QtSql", + windows="Use the installer by Riverbank computing " + "or the standalone qutebrowser exe. " + "http://www.riverbankcomputing.co.uk/" + "software/pyqt/download5", + pip="PyQt5"), } if backend == 'webengine': modules['PyQt5.QtWebEngineWidgets'] = _missing_str("QtWebEngine", @@ -393,6 +400,12 @@ def check_optimize_flag(): "unexpected behavior may occur.") +def check_sqlite(): + from PyQt5.QtSql import QSqlDatabase + if not QSqlDatabase.isDriverAvailable('SQLITE'): + _die('sqlite driver not available! Is sqlite installed?') + + def set_backend(backend): """Set the objects.backend global to the given backend (as string).""" from qutebrowser.misc import objects @@ -432,4 +445,5 @@ def earlyinit(args): check_libraries(backend) check_ssl_support(backend) check_optimize_flag() + check_sqlite() set_backend(backend) From a34df342081a44c58ab2f1634d02aece97a43c22 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 2 Jul 2017 15:22:55 -0400 Subject: [PATCH 159/337] Fix various test/flake8/pylint errors. --- .../completion/models/completionmodel.py | 1 - qutebrowser/completion/models/miscmodels.py | 1 - tests/unit/completion/test_completionmodel.py | 4 +- tests/unit/completion/test_listcategory.py | 39 +++++++------------ 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index a97af2342..778e0cac9 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -22,7 +22,6 @@ from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from qutebrowser.utils import log, qtutils -from qutebrowser.commands import cmdexc class CompletionModel(QAbstractItemModel): diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 00fabb031..167eccde8 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -92,7 +92,6 @@ def buffer(): Used for switching the buffer command. """ - def delete_buffer(data): """Close the selected tab.""" win_id, tab_index = data[0].split('/') diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 5f4670f97..3439400cb 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -27,7 +27,7 @@ import pytest from PyQt5.QtCore import QModelIndex from qutebrowser.completion.models import completionmodel, listcategory -from qutebrowser.commands import cmdexc +from qutebrowser.utils import qtutils @hypothesis.given(strategies.lists(min_size=0, max_size=3, @@ -91,5 +91,5 @@ def test_delete_cur_item(): def test_delete_cur_item_no_cat(): """Test completion_item_del with no selected category.""" model = completionmodel.CompletionModel() - with pytest.raises(cmdexc.CommandError, match='No category selected'): + with pytest.raises(qtutils.QtValueError): model.delete_cur_item(QModelIndex()) diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index d8d35ea3c..377d9abb7 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -25,35 +25,26 @@ from helpers import utils from qutebrowser.completion.models import listcategory -@pytest.mark.parametrize('pattern, filter_cols, before, after', [ - ('foo', [0], - [('foo', '', ''), ('bar', '', '')], - [('foo', '', '')]), +@pytest.mark.parametrize('pattern, before, after', [ + ('foo', + [('foo', ''), ('bar', '')], + [('foo', '')]), - ('foo', [0], - [('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')], - [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]), + ('foo', + [('foob', ''), ('fooc', ''), ('fooa', '')], + [('fooa', ''), ('foob', ''), ('fooc', '')]), # prefer foobar as it starts with the pattern - ('foo', [0], - [('barfoo', '', ''), ('foobar', '', '')], - [('foobar', '', ''), ('barfoo', '', '')]), + ('foo', + [('barfoo', ''), ('foobar', '')], + [('foobar', ''), ('barfoo', '')]), - ('foo', [1], - [('foo', 'bar', ''), ('bar', 'foo', '')], - [('bar', 'foo', '')]), - - ('foo', [0, 1], - [('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')], - [('foo', 'bar', ''), ('bar', 'foo', '')]), - - ('foo', [0, 1, 2], - [('foo', '', ''), ('bar', '')], - [('foo', '', '')]), + ('foo', + [('foo', 'bar'), ('bar', 'foo'), ('bar', 'bar')], + [('foo', 'bar'), ('bar', 'foo')]), ]) -def test_set_pattern(pattern, filter_cols, before, after): +def test_set_pattern(pattern, before, after): """Validate the filtering and sorting results of set_pattern.""" - cat = listcategory.ListCategory('Foo', before, - columns_to_filter=filter_cols) + cat = listcategory.ListCategory('Foo', before) cat.set_pattern(pattern) utils.validate_model(cat, after) From 25c79bec675320f0a1accb2042d5c06e566a9f51 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 3 Jul 2017 08:15:06 -0400 Subject: [PATCH 160/337] Check correct SQL driver in earlyinit. --- qutebrowser/misc/earlyinit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index 95f61708a..eaa493cad 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -402,7 +402,7 @@ def check_optimize_flag(): def check_sqlite(): from PyQt5.QtSql import QSqlDatabase - if not QSqlDatabase.isDriverAvailable('SQLITE'): + if not QSqlDatabase.isDriverAvailable('QSQLITE'): _die('sqlite driver not available! Is sqlite installed?') From 1e1335aa5edf72076a9e25fd129d6e074cba4af6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 3 Jul 2017 08:20:31 -0400 Subject: [PATCH 161/337] Make various SQL code review changes. - Fix outdated comments - Use mock specs when possible - More precise error message check in test_import_txt_invalid. - Fix copyright message - Tweak missing pyqt error message - Dead code: remove group_by and where from sqlcategory. With the new separate completion table, these are no longer used. - Move test_history out of webkit/. History is no longer purely webkit related, it could be webengine. --- qutebrowser/browser/qutescheme.py | 1 - qutebrowser/completion/models/listcategory.py | 10 ++++---- qutebrowser/completion/models/sqlcategory.py | 7 +----- qutebrowser/misc/earlyinit.py | 7 +----- qutebrowser/misc/sql.py | 2 +- .../unit/browser/{webkit => }/test_history.py | 11 ++++++--- tests/unit/completion/test_completionmodel.py | 11 +++++---- .../unit/completion/test_completionwidget.py | 6 ++--- tests/unit/completion/test_listcategory.py | 2 +- tests/unit/completion/test_sqlcategory.py | 23 +------------------ tests/unit/misc/test_lineparser.py | 3 ++- 11 files changed, 29 insertions(+), 54 deletions(-) rename tests/unit/browser/{webkit => }/test_history.py (97%) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 0c495d28b..e25b7df07 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -194,7 +194,6 @@ def history_data(start_time, offset=None): """ hist = objreg.get('web-history') if offset is not None: - # end is 24hrs earlier than start entries = hist.entries_before(start_time, limit=1000, offset=offset) else: # end is 24hrs earlier than start diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index ee9a5e53e..187ebcad6 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -36,7 +36,7 @@ class ListCategory(QSortFilterProxyModel): super().__init__(parent) self.name = name self.srcmodel = QStandardItemModel(parent=self) - self.pattern = '' + self._pattern = '' # ListCategory filters all columns self.columns_to_filter = [0, 1, 2] self.setFilterKeyColumn(-1) @@ -51,7 +51,7 @@ class ListCategory(QSortFilterProxyModel): Args: val: The value to set. """ - self.pattern = val + self._pattern = val val = re.sub(r' +', r' ', val) # See #1919 val = re.escape(val) val = val.replace(r'\ ', '.*') @@ -64,7 +64,7 @@ class ListCategory(QSortFilterProxyModel): def lessThan(self, lindex, rindex): """Custom sorting implementation. - Prefers all items which start with self.pattern. Other than that, uses + Prefers all items which start with self._pattern. Other than that, uses normal Python string sorting. Args: @@ -80,8 +80,8 @@ class ListCategory(QSortFilterProxyModel): left = self.srcmodel.data(lindex) right = self.srcmodel.data(rindex) - leftstart = left.startswith(self.pattern) - rightstart = right.startswith(self.pattern) + leftstart = left.startswith(self._pattern) + rightstart = right.startswith(self._pattern) if leftstart and rightstart: return left < right diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 001a844b0..4819e940b 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -33,7 +33,7 @@ class SqlCategory(QSqlQueryModel): """Wraps a SqlQuery for use as a completion category.""" def __init__(self, name, *, title=None, filter_fields, sort_by=None, - sort_order=None, select='*', where=None, group_by=None, + sort_order=None, select='*', delete_func=None, parent=None): """Create a new completion category backed by a sql table. @@ -42,7 +42,6 @@ class SqlCategory(QSqlQueryModel): title: Title of category, defaults to table name. filter_fields: Names of fields to apply filter pattern to. select: A custom result column expression for the select statement. - where: An optional clause to filter out some rows. sort_by: The name of the field to sort by, or None for no sorting. sort_order: Either 'asc' or 'desc', if sort_by is non-None delete_func: Callback to delete a selected item. @@ -57,10 +56,6 @@ class SqlCategory(QSqlQueryModel): for f in filter_fields) querystr += ')' - if where: - querystr += ' and ' + where - if group_by: - querystr += ' group by {}'.format(group_by) if sort_by: assert sort_order in ['asc', 'desc'], sort_order querystr += ' order by {} {}'.format(sort_by, sort_order) diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index eaa493cad..21be8ed37 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -337,12 +337,7 @@ def check_libraries(backend): "or Install via pip.", pip="PyYAML"), 'PyQt5.QtSql': - _missing_str("PyQt5.QtSql", - windows="Use the installer by Riverbank computing " - "or the standalone qutebrowser exe. " - "http://www.riverbankcomputing.co.uk/" - "software/pyqt/download5", - pip="PyQt5"), + _missing_str("PyQt5.QtSql") } if backend == 'webengine': modules['PyQt5.QtWebEngineWidgets'] = _missing_str("QtWebEngine", diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 4db3fb899..35c5359dd 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -209,8 +209,8 @@ class SqlTable(QObject): """Performantly append multiple rows to the table. Args: - rows: A list of lists, where each sub-list is a row. values: A dict with a list of values to insert for each field name. + replace: If true, overwrite rows with a primary key match. """ q = self._insert_query(values, replace) for key, val in values.items(): diff --git a/tests/unit/browser/webkit/test_history.py b/tests/unit/browser/test_history.py similarity index 97% rename from tests/unit/browser/webkit/test_history.py rename to tests/unit/browser/test_history.py index 4b317acab..9496accd2 100644 --- a/tests/unit/browser/webkit/test_history.py +++ b/tests/unit/browser/test_history.py @@ -113,7 +113,8 @@ def test_clear(qtbot, tmpdir, hist, mocker): hist.add_url(QUrl('http://example.com/')) hist.add_url(QUrl('http://www.qutebrowser.org/')) - m = mocker.patch('qutebrowser.browser.history.message.confirm_async') + m = mocker.patch('qutebrowser.browser.history.message.confirm_async', + spec=[]) hist.clear() assert m.called @@ -288,15 +289,19 @@ def test_import_txt_skip(hist, data_tmpdir, line, monkeypatch, stubs): '68891-x http://example.com/bad-flag', '68891 http://.com', ]) -def test_import_txt_invalid(hist, data_tmpdir, line, monkeypatch, stubs): +def test_import_txt_invalid(hist, data_tmpdir, line, monkeypatch, stubs, + caplog): """import_txt should fail on certain lines.""" monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer) histfile = data_tmpdir / 'history' histfile.write(line) - with pytest.raises(Exception): + with caplog.at_level(logging.ERROR): hist.import_txt() + assert any(rec.msg.startswith("Failed to import history:") + for rec in caplog.records) + assert histfile.exists() diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 3439400cb..bc7f15ab8 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -36,7 +36,7 @@ def test_first_last_item(counts): """Test that first() and last() index to the first and last items.""" model = completionmodel.CompletionModel() for c in counts: - cat = mock.Mock() + cat = mock.Mock(spec=['layoutChanged']) cat.rowCount = mock.Mock(return_value=c) model.add_category(cat) nonempty = [i for i, rowCount in enumerate(counts) if rowCount > 0] @@ -70,16 +70,17 @@ def test_count(counts): def test_set_pattern(pat): """Validate the filtering and sorting results of set_pattern.""" model = completionmodel.CompletionModel() - cats = [mock.Mock(spec=['set_pattern', 'layoutChanged'])] * 3 + cats = [mock.Mock(spec=['set_pattern', 'layoutChanged']) for _ in range(3)] for c in cats: - c.set_pattern = mock.Mock() + c.set_pattern = mock.Mock(spec=[]) model.add_category(c) model.set_pattern(pat) - assert all(c.set_pattern.called_with([pat]) for c in cats) + for c in cats: + c.set_pattern.assert_called_with(pat) def test_delete_cur_item(): - func = mock.Mock() + func = mock.Mock(spec=[]) model = completionmodel.CompletionModel() cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=func) model.add_category(cat) diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 1625dc60a..cad45b5c1 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -83,7 +83,7 @@ def test_set_model(completionview): def test_set_pattern(completionview): model = completionmodel.CompletionModel() - model.set_pattern = mock.Mock() + model.set_pattern = mock.Mock(spec=[]) completionview.set_model(model) completionview.set_pattern('foo') model.set_pattern.assert_called_with('foo') @@ -214,7 +214,7 @@ def test_completion_show(show, rows, quick_complete, completionview, def test_completion_item_del(completionview): """Test that completion_item_del invokes delete_cur_item in the model.""" - func = mock.Mock() + func = mock.Mock(spec=[]) model = completionmodel.CompletionModel() cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=func) model.add_category(cat) @@ -226,7 +226,7 @@ def test_completion_item_del(completionview): def test_completion_item_del_no_selection(completionview): """Test that completion_item_del with an invalid index.""" - func = mock.Mock() + func = mock.Mock(spec=[]) model = completionmodel.CompletionModel() cat = listcategory.ListCategory('', [('foo',)], delete_func=func) model.add_category(cat) diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index 377d9abb7..5b6a5eea4 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2015-2016 Florian Bruhin (The Compiler) +# Copyright 2017 Ryan Roden-Corrent (rcorre) # # This file is part of qutebrowser. # diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 951a7d865..10d96f571 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -135,32 +135,11 @@ def test_select(): utils.validate_model(cat, [('bar', 'baz', 'foo')]) -def test_where(): - table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert({'a': 'foo', 'b': 'bar', 'c': False}) - table.insert({'a': 'baz', 'b': 'biz', 'c': True}) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], where='not c') - cat.set_pattern('') - utils.validate_model(cat, [('foo', 'bar', False)]) - - -def test_group(): - table = sql.SqlTable('Foo', ['a', 'b']) - table.insert({'a': 'foo', 'b': 1}) - table.insert({'a': 'bar', 'b': 3}) - table.insert({'a': 'foo', 'b': 2}) - table.insert({'a': 'bar', 'b': 0}) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], - select='a, max(b)', group_by='a') - cat.set_pattern('') - utils.validate_model(cat, [('bar', 3), ('foo', 2)]) - - def test_delete_cur_item(): table = sql.SqlTable('Foo', ['a', 'b']) table.insert({'a': 'foo', 'b': 1}) table.insert({'a': 'bar', 'b': 2}) - func = unittest.mock.MagicMock() + func = unittest.mock.Mock(spec=[]) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], delete_func=func) cat.set_pattern('') cat.delete_cur_item(cat.index(0, 0)) diff --git a/tests/unit/misc/test_lineparser.py b/tests/unit/misc/test_lineparser.py index cd4a64274..0c78035b7 100644 --- a/tests/unit/misc/test_lineparser.py +++ b/tests/unit/misc/test_lineparser.py @@ -58,7 +58,8 @@ class TestBaseLineParser: mocker.patch('builtins.open', mock.mock_open()) with lineparser._open('r'): - with pytest.raises(IOError): + with pytest.raises(IOError, + match="Refusing to double-open LineParser."): with lineparser._open('r'): pass From 7eacea1057c78e1f29c0ea6934e5fbecbf92c297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Klinkovsk=C3=BD?= Date: Tue, 4 Jul 2017 14:12:21 +0200 Subject: [PATCH 162/337] Preserve fullscreen when exiting video fullscreen Fixes #2778 --- qutebrowser/browser/webengine/webenginetab.py | 1 + qutebrowser/mainwindow/mainwindow.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 0a4c2dfc7..494ba840c 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -615,6 +615,7 @@ class WebEngineTab(browsertab.AbstractTab): def shutdown(self): self.shutting_down.emit() + self.action.exit_fullscreen() if qtutils.version_check('5.8', exact=True): # WORKAROUND for # https://bugreports.qt.io/browse/QTBUG-58563 diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index 067b536bf..5ea212b18 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -128,6 +128,8 @@ class MainWindow(QWidget): _commandrunner: The main CommandRunner instance. _overlays: Widgets shown as overlay for the current webpage. _private: Whether the window is in private browsing mode. + _restore_fullscreen: Whether to restore the fullscreen after leaving + a video fullscreen. """ def __init__(self, *, private, geometry=None, parent=None): @@ -217,6 +219,8 @@ class MainWindow(QWidget): objreg.get("app").new_window.emit(self) + self._restore_fullscreen = False + def _init_geometry(self, geometry): """Initialize the window geometry or load it from disk.""" if geometry is not None: @@ -483,9 +487,14 @@ class MainWindow(QWidget): @pyqtSlot(bool) def _on_fullscreen_requested(self, on): if on: + self._restore_fullscreen = self.isFullScreen() self.showFullScreen() - else: + elif not self._restore_fullscreen: self.showNormal() + else: + self._restore_fullscreen = self.isFullScreen() + log.misc.debug('on: {}, restore fullscreen: {}' + .format(on, self._restore_fullscreen)) @cmdutils.register(instance='main-window', scope='window') @pyqtSlot() From 023bf826384d58e259ce38ce430edb87508d176f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 17:30:47 +0200 Subject: [PATCH 163/337] Update for PyQt 5.9.1 --- .travis.yml | 8 ++++++-- README.asciidoc | 4 ++-- misc/requirements/requirements-pyqt.txt | 4 ++-- tox.ini | 22 ++++++++++++++++++++++ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e18bd2efa..ec2868730 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,14 +23,18 @@ matrix: language: python python: 3.6 env: TESTENV=py36-pyqt571 + - os: linux + language: python + python: 3.6 + env: TESTENV=py36-pyqt58 - os: linux language: python python: 3.5 - env: TESTENV=py35-pyqt58 + env: TESTENV=py35-pyqt59 - os: linux language: python python: 3.6 - env: TESTENV=py36-pyqt58 + env: TESTENV=py36-pyqt59 - os: osx env: TESTENV=py36 OSX=elcapitan osx_image: xcode7.3 diff --git a/README.asciidoc b/README.asciidoc index 347b8356b..1c7e42c33 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -99,10 +99,10 @@ Requirements The following software and libraries are required to run qutebrowser: * http://www.python.org/[Python] 3.4 or newer (3.5 recommended) -* http://qt.io/[Qt] 5.2.0 or newer (5.9.0 recommended) +* http://qt.io/[Qt] 5.2.0 or newer (5.9 recommended) * QtWebKit (old or link:https://github.com/annulen/webkit/wiki[reloaded]/NG) or QtWebEngine * http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.2.0 or newer -(5.8.1 recommended) for Python 3 +(5.9 recommended) for Python 3 * https://pypi.python.org/pypi/setuptools/[pkg_resources/setuptools] * http://fdik.org/pyPEG/[pyPEG2] * http://jinja.pocoo.org/[jinja2] diff --git a/misc/requirements/requirements-pyqt.txt b/misc/requirements/requirements-pyqt.txt index da611589a..fffa133ab 100644 --- a/misc/requirements/requirements-pyqt.txt +++ b/misc/requirements/requirements-pyqt.txt @@ -1,4 +1,4 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -PyQt5==5.8.2 -sip==4.19.2 +PyQt5==5.9 +sip==4.19.3 diff --git a/tox.ini b/tox.ini index 35bcb266c..04cf826d1 100644 --- a/tox.ini +++ b/tox.ini @@ -111,6 +111,28 @@ deps = PyQt5==5.8.2 commands = {envpython} -bb -m pytest {posargs:tests} +[testenv:py35-pyqt59] +basepython = python3.5 +setenv = + {[testenv]setenv} + QUTE_BDD_WEBENGINE=true +passenv = {[testenv]passenv} +deps = + {[testenv]deps} + PyQt5==5.9 +commands = {envpython} -bb -m pytest {posargs:tests} + +[testenv:py36-pyqt59] +basepython = {env:PYTHON:python3.6} +setenv = + {[testenv]setenv} + QUTE_BDD_WEBENGINE=true +passenv = {[testenv]passenv} +deps = + {[testenv]deps} + PyQt5==5.9 +commands = {envpython} -bb -m pytest {posargs:tests} + # other envs [testenv:mkvenv] From 7ea7a2f3fd029fc58e78965151974262de2d567a Mon Sep 17 00:00:00 2001 From: Christian Helbling Date: Tue, 4 Jul 2017 17:50:07 +0200 Subject: [PATCH 164/337] restore maximized state on :fullscreen and when exiting video fullscreen --- qutebrowser/browser/commands.py | 6 +++++- qutebrowser/mainwindow/mainwindow.py | 13 ++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 3d8516fe2..e3b7398ed 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -2171,6 +2171,10 @@ class CommandDispatcher: window = self._tabbed_browser.window() if window.isFullScreen(): - window.showNormal() + if window._restore_maximized: + window.showMaximized() + else: + window.showNormal() else: + window._restore_maximized = window.isMaximized() window.showFullScreen() diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index 5ea212b18..d8fe052b1 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -130,6 +130,8 @@ class MainWindow(QWidget): _private: Whether the window is in private browsing mode. _restore_fullscreen: Whether to restore the fullscreen after leaving a video fullscreen. + _restore_maximized: Whether to restore maximized window after leaving + a video fullscreen. """ def __init__(self, *, private, geometry=None, parent=None): @@ -220,6 +222,7 @@ class MainWindow(QWidget): objreg.get("app").new_window.emit(self) self._restore_fullscreen = False + self._restore_maximized = self.isMaximized() def _init_geometry(self, geometry): """Initialize the window geometry or load it from disk.""" @@ -488,13 +491,17 @@ class MainWindow(QWidget): def _on_fullscreen_requested(self, on): if on: self._restore_fullscreen = self.isFullScreen() + self._restore_maximized = self.isMaximized() self.showFullScreen() elif not self._restore_fullscreen: - self.showNormal() + if self._restore_maximized: + self.showMaximized() + else: + self.showNormal() else: self._restore_fullscreen = self.isFullScreen() - log.misc.debug('on: {}, restore fullscreen: {}' - .format(on, self._restore_fullscreen)) + log.misc.debug('on: {}, restore fullscreen: {}, restore maximized: {}' + .format(on, self._restore_fullscreen, self._restore_maximized)) @cmdutils.register(instance='main-window', scope='window') @pyqtSlot() From 378914b327c9d4be8afdb029b314719014b7b2f2 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 18:01:11 +0200 Subject: [PATCH 165/337] Ignore another new geoclue error during tests --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index a2d2779d9..ab437801b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -45,6 +45,7 @@ qt_log_ignore = ^QStandardPaths: XDG_RUNTIME_DIR not set, defaulting to .* ^QGeoclueMaster error creating GeoclueMasterClient\. ^Geoclue error: Process org\.freedesktop\.Geoclue\.Master exited with status 127 + ^QDBusConnection: name 'org.freedesktop.Geoclue.Master' had owner '' but we thought it was ':1.1' ^QObject::connect: Cannot connect \(null\)::stateChanged\(QNetworkSession::State\) to QNetworkReplyHttpImpl::_q_networkSessionStateChanged\(QNetworkSession::State\) ^QXcbClipboard: Cannot transfer data, no data available ^load glyph failed From 28410b85339acd2708b864568981c83b4511da4d Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 18:01:47 +0200 Subject: [PATCH 166/337] Release v0.11.0 --- CHANGELOG.asciidoc | 102 +++---- CONTRIBUTING.asciidoc | 3 +- qutebrowser/__init__.py | 2 +- .../old_configs/qutebrowser-v0.11.0.conf | 251 ++++++++++++++++++ tests/unit/config/test_config.py | 2 +- 5 files changed, 306 insertions(+), 54 deletions(-) create mode 100644 tests/unit/config/old_configs/qutebrowser-v0.11.0.conf diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index ea7481030..14fd41b88 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -14,8 +14,8 @@ This project adheres to http://semver.org/[Semantic Versioning]. // `Fixed` for any bug fixes. // `Security` to invite users to upgrade in case of vulnerabilities. -v0.11.0 (unreleased) --------------------- +v0.11.0 +------- New dependencies ~~~~~~~~~~~~~~~~ @@ -28,7 +28,10 @@ New dependencies Added ~~~~~ -- New `-p` flag for `:open` to open a private window. +- Private browsing is now implemented for QtWebEngine, *and changed its + behavior*: The `general -> private-browsing` setting now only applies to newly + opened windows, and you can use the `-p` flag to `:open` to open a private + window. - New "pinned tabs" feature, with a new `:tab-pin` command (bound to `` by default). - (QtWebEngine) Implemented `:follow-selected`. @@ -45,7 +48,8 @@ Added customize statusbar colors for private windows. - New `{private}` field displaying `[Private Mode]` for `ui -> window-title-format` and `tabs -> title-format`. -- (QtWebEngine) Proxy support with Qt 5.7.1 too +- (QtWebEngine) Proxy support with Qt 5.7.1 (already was supported for 5.8 and + newer) Changed ~~~~~~~ @@ -53,62 +57,51 @@ Changed - To prevent elaborate phishing attacks, the Punycode version (`xn--*`) is now shown in addition to the decoded version for international domain names (IDN). -- Private browsing is now implemented for QtWebEngine, and changed it's - behavior: The `general -> private-browsing` setting now only applies to newly - opened windows, and you can use the `-p` flag to `:open` to open a private - window. -- Improved `qute://history` page (with lazy loading) -- Starting with legacy QtWebKit now shows a warning message once. -- Crash reports are not public anymore. -- Paths like `C:` are now treated as absolute paths on Windows for downloads, - and invalid paths are handled properly. -- PAC on QtWebKit now supports SOCKS5 as type. -- Comments in the config file are now before the individual options instead of - being before sections. -- Messages are now hidden when clicked. -- stdin is now closed immediately for processes spawned from qutebrowser. -- When `ui -> message-timeout` is set to 0, messages are now never cleared. -- Middle/right-clicking the blank parts of the tab bar (when vertical) now - closes the current tab. -- (QtWebEngine) With Qt 5.9, `content -> cookies-store` can now be set without - a restart. -- (QtWebEngine) With Qt 5.9, better error messages are now shown for failed - downloads. -- The adblocker now also blocks non-GET requests (e.g. POST). -- `javascript:` links can now be hinted. -- `:view-source`, `:tab-clone` and `:navigate --tab` now don't open the tab as - "explicit" anymore, i.e. (with the default settings) open it next to the - active tab. -- (QtWebEngine) The underlying Chromium version is now shown in the version - info. -- `qute:*` pages now use `qute://*` instead (e.g. `qute://version` instead of - `qute:version`), but the old versions are automatically redirected. +- Starting with legacy QtWebKit now shows a warning message. + *With the next release, support for it will be removed.* - The Windows releases are redone from scratch, which means: - They now use the new QtWebEngine backend - The bundled Qt is updated from 5.5 to 5.9 - The bundled Python is updated from 3.4 to 3.6 - They are now generated with PyInstaller instead of cx_Freeze - The installer is now generated using NSIS instead of being a MSI +- Improved `qute://history` page (with lazy loading) +- Crash reports are not public anymore. +- Paths like `C:` are now treated as absolute paths on Windows for downloads, + and invalid paths are handled properly. +- Comments in the config file are now placed before the individual options + instead of being before sections. +- Messages are now hidden when clicked. +- stdin is now closed immediately for processes spawned from qutebrowser. +- When `ui -> message-timeout` is set to 0, messages are now never cleared. +- Middle/right-clicking the blank parts of the tab bar (when vertical) now + closes the current tab. +- The adblocker now also blocks non-GET requests (e.g. POST). +- `javascript:` links can now be hinted. +- `:view-source`, `:tab-clone` and `:navigate --tab` now don't open the tab as + "explicit" anymore, i.e. (with the default settings) open it next to the + active tab. +- `qute:*` pages now use `qute://*` instead (e.g. `qute://version` instead of + `qute:version`), but the old versions are automatically redirected. - Texts in prompts are now selectable. -- Renderer process crashes now show an error page. -- (QtWebKit) storage -> offline-web-application-storage` got renamed to `...-cache` - The default level for `:messages` is now `info`, not `error` - Trying to focus the currently focused tab with `:tab-focus` now focuses the last viewed tab. +- (QtWebEngine) With Qt 5.9, `content -> cookies-store` can now be set without + a restart. +- (QtWebEngine) With Qt 5.9, better error messages are now shown for failed + downloads. +- (QtWebEngine) The underlying Chromium version is now shown in the version + info. +- (QtWebKit) Renderer process crashes now show an error page on Qt 5.9 or newer. +- (QtWebKit) storage -> offline-web-application-storage` got renamed to `...-cache` +- (QtWebKit) PAC now supports SOCKS5 as type. Fixed ~~~~~ - The macOS .dmg is now built against Qt 5.9 which fixes various important issues (such as not being able to type dead keys). -- (QtWebEngine) Added a workaround for a black screen with some setups - (the workaround requires PyOpenGL to be installed, but it's optional) -- (QtWebEngine) Starting with Nouveau graphics now shows an error message - instead of crashing in Qt. This adds a new dependency on `PyQt5.QtOpenGL`. -- (QtWebEngine) Retrying downloads now shows an error instead of crashing. -- (QtWebEngine) Cloning a view-source tab now doesn't crash anymore. -- (QtWebKit) The HTTP cache is disabled on Qt 5.7.1 and 5.8 now as it leads to - frequent crashes due to a Qt bug. - Fixed crash with `:download` on PyQt 5.9. - Cloning a page without history doesn't crash anymore. - When a download results in a HTTP error, it now shows the error correctly @@ -118,7 +111,6 @@ Fixed - Fixed crash when unbinding an unbound key in the key config. - Fixed crash when using `:debug-log-filter` when `--filter` wasn't given on startup. - Fixed crash with some invalid setting values. -- (QtWebKit) Fixed Crash when a PAC file returns an invalid value. - Continuing a search after clearing it now works correctly. - The tabbar and completion should now be more consistently and correctly styled with various system styles. @@ -126,19 +118,27 @@ Fixed - The validation for colors in stylesheets is now less strict, allowing for all valid Qt values. - `data:` URLs now aren't added to the history anymore. -- (QtWebEngine) `window.navigator.userAgent` is now set correctly when - customizing the user agent. - Accidentally starting with Python 2 now shows a proper error message again. -- (QtWebEngine) HTML fullscreen is now tracked for each tab separately, which - means it's not possible anymore to accidentally get stuck in fullscreen state - by closing a tab with a fullscreen video. - For some people, running some userscripts crashed - this should now be fixed. - Various other rare crashes should now be fixed. - The settings documentation was truncated with v0.10.1 which should now be fixed. - Scrolling to an anchor in a background tab now works correctly, and javascript gets the correct window size for background tabs. -- (QtWebEngine) `:scroll-page` with `--bottom-navigate` now works correctly +- (QtWebEngine) Added a workaround for a black screen with some setups +- (QtWebEngine) Starting with Nouveau graphics now shows an error message + instead of crashing in Qt. +- (QtWebEngine) Retrying downloads now shows an error instead of crashing. +- (QtWebEngine) Cloning a view-source tab now doesn't crash anymore. +- (QtWebEngine) `window.navigator.userAgent` is now set correctly when + customizing the user agent. +- (QtWebEngine) HTML fullscreen is now tracked for each tab separately, which + means it's not possible anymore to accidentally get stuck in fullscreen state + by closing a tab with a fullscreen video. +- (QtWebEngine) `:scroll-page` with `--bottom-navigate` now works correctly. +- (QtWebKit) The HTTP cache is disabled on Qt 5.7.1 and 5.8 now as it leads to + frequent crashes due to a Qt bug. +- (QtWebKit) Fixed Crash when a PAC file returns an invalid value. v0.10.1 ------- diff --git a/CONTRIBUTING.asciidoc b/CONTRIBUTING.asciidoc index d5c77f521..5a7eafd77 100644 --- a/CONTRIBUTING.asciidoc +++ b/CONTRIBUTING.asciidoc @@ -682,8 +682,9 @@ qutebrowser release * Add newest config to `tests/unit/config/old_configs` and update `test_upgrade_version` - `python -m qutebrowser --basedir conf :quit` - - `sed '/^#/d' conf/config/qutebrowser.conf > tests/unit/config/old_configs/qutebrowser-v0.x.y.conf` + - `sed '/^#/d' conf/config/qutebrowser.conf > tests/unit/config/old_configs/qutebrowser-v0.$x.$y.conf` - `rm -r conf` + - git add - commit * Adjust `__version_info__` in `qutebrowser/__init__.py`. * Update changelog (remove *(unreleased)*) diff --git a/qutebrowser/__init__.py b/qutebrowser/__init__.py index e61419c0c..cca2bf1b8 100644 --- a/qutebrowser/__init__.py +++ b/qutebrowser/__init__.py @@ -26,7 +26,7 @@ __copyright__ = "Copyright 2014-2017 Florian Bruhin (The Compiler)" __license__ = "GPL" __maintainer__ = __author__ __email__ = "mail@qutebrowser.org" -__version_info__ = (0, 10, 1) +__version_info__ = (0, 11, 0) __version__ = '.'.join(str(e) for e in __version_info__) __description__ = "A keyboard-driven, vim-like browser based on PyQt5." diff --git a/tests/unit/config/old_configs/qutebrowser-v0.11.0.conf b/tests/unit/config/old_configs/qutebrowser-v0.11.0.conf new file mode 100644 index 000000000..ba48a6ff5 --- /dev/null +++ b/tests/unit/config/old_configs/qutebrowser-v0.11.0.conf @@ -0,0 +1,251 @@ +[general] +ignore-case = smart +startpage = https://start.duckduckgo.com +yank-ignored-url-parameters = ref,utm_source,utm_medium,utm_campaign,utm_term,utm_content +default-open-dispatcher = +default-page = ${startpage} +auto-search = naive +auto-save-config = true +auto-save-interval = 15000 +editor = gvim -f "{}" +editor-encoding = utf-8 +private-browsing = false +developer-extras = false +print-element-backgrounds = true +xss-auditing = false +default-encoding = iso-8859-1 +new-instance-open-target = tab +new-instance-open-target.window = last-focused +log-javascript-console = debug +save-session = false +session-default-name = +url-incdec-segments = path,query +[ui] +history-session-interval = 30 +zoom-levels = 25%,33%,50%,67%,75%,90%,100%,110%,125%,150%,175%,200%,250%,300%,400%,500% +default-zoom = 100% +downloads-position = top +status-position = bottom +message-timeout = 2000 +message-unfocused = false +confirm-quit = never +zoom-text-only = false +frame-flattening = false +user-stylesheet = +hide-scrollbar = true +smooth-scrolling = false +remove-finished-downloads = -1 +hide-statusbar = false +statusbar-padding = 1,1,0,0 +window-title-format = {perc}{title}{title_sep}qutebrowser +modal-js-dialog = false +hide-wayland-decoration = false +keyhint-blacklist = +keyhint-delay = 500 +prompt-radius = 8 +prompt-filebrowser = true +[network] +do-not-track = true +accept-language = en-US,en +referer-header = same-domain +user-agent = +proxy = system +proxy-dns-requests = true +ssl-strict = ask +dns-prefetch = true +custom-headers = +netrc-file = +[completion] +show = always +download-path-suggestion = path +timestamp-format = %Y-%m-%d +height = 50% +cmd-history-max-items = 100 +web-history-max-items = 1000 +quick-complete = true +shrink = false +scrollbar-width = 12 +scrollbar-padding = 2 +[input] +timeout = 500 +partial-timeout = 5000 +insert-mode-on-plugins = false +auto-leave-insert-mode = true +auto-insert-mode = false +forward-unbound-keys = auto +spatial-navigation = false +links-included-in-focus-chain = true +rocker-gestures = false +mouse-zoom-divider = 512 +[tabs] +background-tabs = false +select-on-remove = next +new-tab-position = next +new-tab-position-explicit = last +last-close = ignore +show = always +show-switching-delay = 800 +wrap = true +movable = true +close-mouse-button = middle +position = top +show-favicons = true +favicon-scale = 1.0 +width = 20% +pinned-width = 43 +indicator-width = 3 +tabs-are-windows = false +title-format = {index}: {title} +title-format-pinned = {index} +title-alignment = left +mousewheel-tab-switching = true +padding = 0,0,5,5 +indicator-padding = 2,2,0,4 +[storage] +download-directory = +prompt-download-directory = true +remember-download-directory = true +maximum-pages-in-cache = 0 +offline-web-application-cache = true +local-storage = true +cache-size = +[content] +allow-images = true +allow-javascript = true +allow-plugins = false +webgl = true +hyperlink-auditing = false +geolocation = ask +notifications = ask +media-capture = ask +javascript-can-open-windows-automatically = false +javascript-can-close-windows = false +javascript-can-access-clipboard = false +ignore-javascript-prompt = false +ignore-javascript-alert = false +local-content-can-access-remote-urls = false +local-content-can-access-file-urls = true +cookies-accept = no-3rdparty +cookies-store = true +host-block-lists = https://www.malwaredomainlist.com/hostslist/hosts.txt,http://someonewhocares.org/hosts/hosts,http://winhelp2002.mvps.org/hosts.zip,http://malwaredomains.lehigh.edu/files/justdomains.zip,https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&mimetype=plaintext +host-blocking-enabled = true +host-blocking-whitelist = piwik.org +enable-pdfjs = false +[hints] +border = 1px solid #E3BE23 +mode = letter +chars = asdfghjkl +min-chars = 1 +scatter = true +uppercase = false +dictionary = /usr/share/dict/words +auto-follow = unique-match +auto-follow-timeout = 0 +next-regexes = \bnext\b,\bmore\b,\bnewer\b,\b[>→≫]\b,\b(>>|»)\b,\bcontinue\b +prev-regexes = \bprev(ious)?\b,\bback\b,\bolder\b,\b[<←≪]\b,\b(<<|«)\b +find-implementation = python +hide-unmatched-rapid-hints = true +[searchengines] +DEFAULT = https://duckduckgo.com/?q={} +[aliases] +[colors] +completion.fg = white +completion.bg = #333333 +completion.alternate-bg = #444444 +completion.category.fg = white +completion.category.bg = qlineargradient(x1:0, y1:0, x2:0, y2:1, stop:0 #888888, stop:1 #505050) +completion.category.border.top = black +completion.category.border.bottom = ${completion.category.border.top} +completion.item.selected.fg = black +completion.item.selected.bg = #e8c000 +completion.item.selected.border.top = #bbbb00 +completion.item.selected.border.bottom = ${completion.item.selected.border.top} +completion.match.fg = #ff4444 +completion.scrollbar.fg = ${completion.fg} +completion.scrollbar.bg = ${completion.bg} +statusbar.fg = white +statusbar.bg = black +statusbar.fg.private = ${statusbar.fg} +statusbar.bg.private = #666666 +statusbar.fg.insert = ${statusbar.fg} +statusbar.bg.insert = darkgreen +statusbar.fg.command = ${statusbar.fg} +statusbar.bg.command = ${statusbar.bg} +statusbar.fg.command.private = ${statusbar.fg.private} +statusbar.bg.command.private = ${statusbar.bg.private} +statusbar.fg.caret = ${statusbar.fg} +statusbar.bg.caret = purple +statusbar.fg.caret-selection = ${statusbar.fg} +statusbar.bg.caret-selection = #a12dff +statusbar.progress.bg = white +statusbar.url.fg = ${statusbar.fg} +statusbar.url.fg.success = white +statusbar.url.fg.success.https = lime +statusbar.url.fg.error = orange +statusbar.url.fg.warn = yellow +statusbar.url.fg.hover = aqua +tabs.fg.odd = white +tabs.bg.odd = grey +tabs.fg.even = white +tabs.bg.even = darkgrey +tabs.fg.selected.odd = white +tabs.bg.selected.odd = black +tabs.fg.selected.even = ${tabs.fg.selected.odd} +tabs.bg.selected.even = ${tabs.bg.selected.odd} +tabs.bg.bar = #555555 +tabs.indicator.start = #0000aa +tabs.indicator.stop = #00aa00 +tabs.indicator.error = #ff0000 +tabs.indicator.system = rgb +hints.fg = black +hints.bg = qlineargradient(x1:0, y1:0, x2:0, y2:1, stop:0 rgba(255, 247, 133, 0.8), stop:1 rgba(255, 197, 66, 0.8)) +hints.fg.match = green +downloads.bg.bar = black +downloads.fg.start = white +downloads.bg.start = #0000aa +downloads.fg.stop = ${downloads.fg.start} +downloads.bg.stop = #00aa00 +downloads.fg.system = rgb +downloads.bg.system = rgb +downloads.fg.error = white +downloads.bg.error = red +webpage.bg = white +keyhint.fg = #FFFFFF +keyhint.fg.suffix = #FFFF00 +keyhint.bg = rgba(0, 0, 0, 80%) +messages.fg.error = white +messages.bg.error = red +messages.border.error = #bb0000 +messages.fg.warning = white +messages.bg.warning = darkorange +messages.border.warning = #d47300 +messages.fg.info = white +messages.bg.info = black +messages.border.info = #333333 +prompts.fg = white +prompts.bg = darkblue +prompts.selected.bg = #308cc6 +[fonts] +_monospace = xos4 Terminus, Terminus, Monospace, "DejaVu Sans Mono", Monaco, "Bitstream Vera Sans Mono", "Andale Mono", "Courier New", Courier, "Liberation Mono", monospace, Fixed, Consolas, Terminal +completion = 8pt ${_monospace} +completion.category = bold ${completion} +tabbar = 8pt ${_monospace} +statusbar = 8pt ${_monospace} +downloads = 8pt ${_monospace} +hints = bold 13px ${_monospace} +debug-console = 8pt ${_monospace} +web-family-standard = +web-family-fixed = +web-family-serif = +web-family-sans-serif = +web-family-cursive = +web-family-fantasy = +web-size-minimum = 0 +web-size-minimum-logical = 6 +web-size-default = 16 +web-size-default-fixed = 13 +keyhint = 8pt ${_monospace} +messages.error = 8pt ${_monospace} +messages.warning = 8pt ${_monospace} +messages.info = 8pt ${_monospace} +prompts = 8pt sans-serif diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index c897b9f16..be526dac3 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -408,7 +408,7 @@ class TestDefaultConfig: If it did change, place a new qutebrowser-vx.y.z.conf in old_configs and then increment the version. """ - assert qutebrowser.__version__ == '0.10.1' + assert qutebrowser.__version__ == '0.11.0' @pytest.mark.parametrize('filename', os.listdir(os.path.join(os.path.dirname(__file__), 'old_configs')), From defe140d98a6a82b1fba0c1440fb827a758b4733 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 19:56:54 +0200 Subject: [PATCH 167/337] build_release: Run tox with -vv --- scripts/dev/build_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 073b9a58e..9190a8801 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -64,7 +64,7 @@ def call_tox(toxenv, *args, python=sys.executable): env['PYTHON'] = python env['PATH'] = os.environ['PATH'] + os.pathsep + os.path.dirname(python) subprocess.check_call( - [sys.executable, '-m', 'tox', '-v', '-e', toxenv] + list(args), + [sys.executable, '-m', 'tox', '-vv', '-e', toxenv] + list(args), env=env) From 2df9508e44cd6839075c7be725fb13ced4563a08 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 21:03:34 +0200 Subject: [PATCH 168/337] Add PyQt5 OpenGL module to PyInstaller hiddenimports --- misc/qutebrowser.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/qutebrowser.spec b/misc/qutebrowser.spec index 5dc51015d..cd0ce3883 100644 --- a/misc/qutebrowser.spec +++ b/misc/qutebrowser.spec @@ -41,7 +41,7 @@ a = Analysis(['../qutebrowser/__main__.py'], pathex=['misc'], binaries=None, datas=get_data_files(), - hiddenimports=['PyQt5.QtOpenGL'], + hiddenimports=['PyQt5.QtOpenGL', 'PyQt5._QOpenGLFunctions_2_0'], hookspath=[], runtime_hooks=[], excludes=['tkinter'], From d96403fe93713c589f88d0894cda6f5f490f82af Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 21:03:55 +0200 Subject: [PATCH 169/337] build_release: Clean up before doing stuff So we can inspect the results later. --- scripts/dev/build_release.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 9190a8801..148266d21 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -124,6 +124,14 @@ def patch_osx_app(): def build_osx(): """Build OS X .dmg/.app.""" + utils.print_title("Cleaning up...") + for f in ['wc.dmg', 'template.dmg']: + try: + os.remove(f) + except FileNotFoundError: + pass + for d in ['dist', 'build']: + shutil.rmtree(d, ignore_errors=True) utils.print_title("Updating 3rdparty content") # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False) utils.print_title("Building .app via pyinstaller") @@ -132,11 +140,6 @@ def build_osx(): patch_osx_app() utils.print_title("Building .dmg") subprocess.check_call(['make', '-f', 'scripts/dev/Makefile-dmg']) - utils.print_title("Cleaning up...") - for f in ['wc.dmg', 'template.dmg']: - os.remove(f) - for d in ['dist', 'build']: - shutil.rmtree(d) dmg_name = 'qutebrowser-{}.dmg'.format(qutebrowser.__version__) os.rename('qutebrowser.dmg', dmg_name) From 7ecdd6c1c5a514ea7b63e042988d21221381ce2a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 21:04:08 +0200 Subject: [PATCH 170/337] build_release: Print some more information about copied files --- scripts/dev/build_release.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 148266d21..dfb5f4a74 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -109,8 +109,11 @@ def patch_osx_app(): for f in glob.glob(os.path.join(qtwe_core_dir, 'Resources', '*')): dest = os.path.join(app_path, 'Contents', 'Resources') if os.path.isdir(f): - shutil.copytree(f, os.path.join(dest, f)) + dir_dest = os.path.join(dest, f) + print("Copying directory {} to {}".format(f, dir_dest)) + shutil.copytree(f, dir_dest) else: + print("Copying {} to {}".format(f, dest)) shutil.copy(f, dest) # Link dependencies for lib in ['QtCore', 'QtWebEngineCore', 'QtQuick', 'QtQml', 'QtNetwork', From 8f03a36862bdb0d288937de7ced3785c4d62df52 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 21:31:53 +0200 Subject: [PATCH 171/337] build_release: Use correct path when copying dirs --- scripts/dev/build_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index dfb5f4a74..f87dd4bdf 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -109,7 +109,7 @@ def patch_osx_app(): for f in glob.glob(os.path.join(qtwe_core_dir, 'Resources', '*')): dest = os.path.join(app_path, 'Contents', 'Resources') if os.path.isdir(f): - dir_dest = os.path.join(dest, f) + dir_dest = os.path.join(dest, os.path.basename(f)) print("Copying directory {} to {}".format(f, dir_dest)) shutil.copytree(f, dir_dest) else: From 3cbe419cee5781e98f8b64739e232d90570367e2 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 21:34:10 +0200 Subject: [PATCH 172/337] Update Python version for Windows in release checklist --- CONTRIBUTING.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.asciidoc b/CONTRIBUTING.asciidoc index 5a7eafd77..2ac99804e 100644 --- a/CONTRIBUTING.asciidoc +++ b/CONTRIBUTING.asciidoc @@ -699,7 +699,7 @@ qutebrowser release as closed. * Linux: Run `python3 scripts/dev/build_release.py --upload v0.$x.$y` -* Windows: Run `C:\Python34_x32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v0.X.Y` (replace X/Y by hand) +* Windows: Run `C:\Python36-32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v0.X.Y` (replace X/Y by hand) * OS X: Run `python3 scripts/dev/build_release.py --upload v0.X.Y` (replace X/Y by hand) * On server: Run `python3 scripts/dev/download_release.py v0.X.Y` (replace X/Y by hand) * Update `qutebrowser-git` PKGBUILD if dependencies/install changed From c424a745d8d039b02f62dc9b545693b9025012e6 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 21:35:06 +0200 Subject: [PATCH 173/337] build_release: Add comment about missing 3rdparty upgrade --- scripts/dev/build_release.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index f87dd4bdf..5c2c5f168 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -136,6 +136,7 @@ def build_osx(): for d in ['dist', 'build']: shutil.rmtree(d, ignore_errors=True) utils.print_title("Updating 3rdparty content") + # Currently disabled because QtWebEngine has no pdfjs support # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False) utils.print_title("Building .app via pyinstaller") call_tox('pyinstaller', '-r') @@ -173,6 +174,7 @@ def patch_windows(out_dir): def build_windows(): """Build windows executables/setups.""" utils.print_title("Updating 3rdparty content") + # Currently disabled because QtWebEngine has no pdfjs support # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False) utils.print_title("Building Windows binaries") From 725d4a44f01de99fab5d9e4404a9123f40ec8915 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 22:16:21 +0200 Subject: [PATCH 174/337] build_release: Don't fail if hdiutil detach fails --- scripts/dev/build_release.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 5c2c5f168..27d7b04b8 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -149,15 +149,19 @@ def build_osx(): os.rename('qutebrowser.dmg', dmg_name) utils.print_title("Running smoke test") - with tempfile.TemporaryDirectory() as tmpdir: - subprocess.check_call(['hdiutil', 'attach', dmg_name, - '-mountpoint', tmpdir]) - try: - binary = os.path.join(tmpdir, 'qutebrowser.app', 'Contents', - 'MacOS', 'qutebrowser') - smoke_test(binary) - finally: - subprocess.check_call(['hdiutil', 'detach', tmpdir]) + + try: + with tempfile.TemporaryDirectory() as tmpdir: + subprocess.check_call(['hdiutil', 'attach', dmg_name, + '-mountpoint', tmpdir]) + try: + binary = os.path.join(tmpdir, 'qutebrowser.app', 'Contents', + 'MacOS', 'qutebrowser') + smoke_test(binary) + finally: + subprocess.call(['hdiutil', 'detach', tmpdir]) + except PermissionError as e: + print("Failed to remove tempdir: {}".format(e)) return [(dmg_name, 'application/x-apple-diskimage', 'OS X .dmg')] From eaecfe5882f59abaecabb688340c88b8638f4b1b Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 4 Jul 2017 22:27:17 +0200 Subject: [PATCH 175/337] build_release: Adjust Windows installer names --- scripts/dev/build_release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 27d7b04b8..e6a4f89e1 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -215,8 +215,8 @@ def build_windows(): '/DVERSION={}'.format(qutebrowser.__version__), 'misc/qutebrowser.nsi']) - name_32 = 'qutebrowser-{}-win32.msi'.format(qutebrowser.__version__) - name_64 = 'qutebrowser-{}-amd64.msi'.format(qutebrowser.__version__) + name_32 = 'qutebrowser-{}-win32.exe'.format(qutebrowser.__version__) + name_64 = 'qutebrowser-{}-amd64.exe'.format(qutebrowser.__version__) artifacts += [ (os.path.join('dist', name_32), From 20db65e430beb7bbe9f6f0b895dcaea34c64cd55 Mon Sep 17 00:00:00 2001 From: Christian Helbling Date: Tue, 4 Jul 2017 22:56:44 +0200 Subject: [PATCH 176/337] preserve window state when exiting video fullscreen --- qutebrowser/browser/commands.py | 7 ++----- qutebrowser/mainwindow/mainwindow.py | 22 ++++++---------------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index e3b7398ed..080c3e209 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -2171,10 +2171,7 @@ class CommandDispatcher: window = self._tabbed_browser.window() if window.isFullScreen(): - if window._restore_maximized: - window.showMaximized() - else: - window.showNormal() + window.setWindowState(window._state_before_fullscreen & ~Qt.WindowFullScreen) else: - window._restore_maximized = window.isMaximized() + window._state_before_fullscreen = window.windowState() window.showFullScreen() diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index d8fe052b1..2fac3f51a 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -128,10 +128,7 @@ class MainWindow(QWidget): _commandrunner: The main CommandRunner instance. _overlays: Widgets shown as overlay for the current webpage. _private: Whether the window is in private browsing mode. - _restore_fullscreen: Whether to restore the fullscreen after leaving - a video fullscreen. - _restore_maximized: Whether to restore maximized window after leaving - a video fullscreen. + _state_before_fullscreen: window state before activation of fullscreen """ def __init__(self, *, private, geometry=None, parent=None): @@ -221,8 +218,7 @@ class MainWindow(QWidget): objreg.get("app").new_window.emit(self) - self._restore_fullscreen = False - self._restore_maximized = self.isMaximized() + self._state_before_fullscreen = self.windowState() def _init_geometry(self, geometry): """Initialize the window geometry or load it from disk.""" @@ -490,18 +486,12 @@ class MainWindow(QWidget): @pyqtSlot(bool) def _on_fullscreen_requested(self, on): if on: - self._restore_fullscreen = self.isFullScreen() - self._restore_maximized = self.isMaximized() + self._state_before_fullscreen = self.windowState() self.showFullScreen() - elif not self._restore_fullscreen: - if self._restore_maximized: - self.showMaximized() - else: - self.showNormal() else: - self._restore_fullscreen = self.isFullScreen() - log.misc.debug('on: {}, restore fullscreen: {}, restore maximized: {}' - .format(on, self._restore_fullscreen, self._restore_maximized)) + self.setWindowState(self._state_before_fullscreen) + log.misc.debug('on: {}, state before fullscreen: {}' + .format(on, self._state_before_fullscreen)) @cmdutils.register(instance='main-window', scope='window') @pyqtSlot() From 361251bf538c61239b8c007b5643ba993dc2e0b9 Mon Sep 17 00:00:00 2001 From: Christian Helbling Date: Tue, 4 Jul 2017 23:30:06 +0200 Subject: [PATCH 177/337] mark public attribute as public, fix debug output --- qutebrowser/browser/commands.py | 8 +++++--- qutebrowser/mainwindow/mainwindow.py | 14 +++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 080c3e209..9f7cef9d4 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -38,7 +38,7 @@ from qutebrowser.browser import (urlmarks, browsertab, inspector, navigate, webelem, downloads) from qutebrowser.keyinput import modeman from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils, - objreg, utils, typing) + objreg, utils, typing, debug) from qutebrowser.utils.usertypes import KeyMode from qutebrowser.misc import editor, guiprocess from qutebrowser.completion.models import instances, sortfilter @@ -2171,7 +2171,9 @@ class CommandDispatcher: window = self._tabbed_browser.window() if window.isFullScreen(): - window.setWindowState(window._state_before_fullscreen & ~Qt.WindowFullScreen) + window.setWindowState(window.state_before_fullscreen & ~Qt.WindowFullScreen) else: - window._state_before_fullscreen = window.windowState() + window.state_before_fullscreen = window.windowState() window.showFullScreen() + log.misc.debug('state before fullscreen: {}' + .format(debug.qflags_key(Qt, window.state_before_fullscreen))) diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index 2fac3f51a..6edabda09 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -30,7 +30,7 @@ from PyQt5.QtWidgets import QWidget, QVBoxLayout, QApplication, QSizePolicy from qutebrowser.commands import runners, cmdutils from qutebrowser.config import config -from qutebrowser.utils import message, log, usertypes, qtutils, objreg, utils +from qutebrowser.utils import message, log, usertypes, qtutils, objreg, utils, debug from qutebrowser.mainwindow import tabbedbrowser, messageview, prompt from qutebrowser.mainwindow.statusbar import bar from qutebrowser.completion import completionwidget, completer @@ -123,12 +123,12 @@ class MainWindow(QWidget): Attributes: status: The StatusBar widget. tabbed_browser: The TabbedBrowser widget. + state_before_fullscreen: window state before activation of fullscreen. _downloadview: The DownloadView widget. _vbox: The main QVBoxLayout. _commandrunner: The main CommandRunner instance. _overlays: Widgets shown as overlay for the current webpage. _private: Whether the window is in private browsing mode. - _state_before_fullscreen: window state before activation of fullscreen """ def __init__(self, *, private, geometry=None, parent=None): @@ -218,7 +218,7 @@ class MainWindow(QWidget): objreg.get("app").new_window.emit(self) - self._state_before_fullscreen = self.windowState() + self.state_before_fullscreen = self.windowState() def _init_geometry(self, geometry): """Initialize the window geometry or load it from disk.""" @@ -486,12 +486,12 @@ class MainWindow(QWidget): @pyqtSlot(bool) def _on_fullscreen_requested(self, on): if on: - self._state_before_fullscreen = self.windowState() + self.state_before_fullscreen = self.windowState() self.showFullScreen() else: - self.setWindowState(self._state_before_fullscreen) - log.misc.debug('on: {}, state before fullscreen: {}' - .format(on, self._state_before_fullscreen)) + self.setWindowState(self.state_before_fullscreen) + log.misc.debug('on: {}, state before fullscreen: {}'.format( + on, debug.qflags_key(Qt, self.state_before_fullscreen))) @cmdutils.register(instance='main-window', scope='window') @pyqtSlot() From 81f5b7115f6600ad8ba178c4b2b1a74f97120b94 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 5 Jul 2017 08:44:56 -0400 Subject: [PATCH 178/337] Add spec=[] to two mock functions in tests. --- tests/unit/completion/test_completionmodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index bc7f15ab8..2c0af3a4b 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -37,7 +37,7 @@ def test_first_last_item(counts): model = completionmodel.CompletionModel() for c in counts: cat = mock.Mock(spec=['layoutChanged']) - cat.rowCount = mock.Mock(return_value=c) + cat.rowCount = mock.Mock(return_value=c, spec=[]) model.add_category(cat) nonempty = [i for i, rowCount in enumerate(counts) if rowCount > 0] if not nonempty: @@ -61,7 +61,7 @@ def test_count(counts): model = completionmodel.CompletionModel() for c in counts: cat = mock.Mock(spec=['rowCount', 'layoutChanged']) - cat.rowCount = mock.Mock(return_value=c) + cat.rowCount = mock.Mock(return_value=c, spec=[]) model.add_category(cat) assert model.count() == sum(counts) From f15dbecc73b33df46b314232740d604520fa46c9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 5 Jul 2017 21:49:34 +0200 Subject: [PATCH 179/337] Update changelog for unreleased versions --- CHANGELOG.asciidoc | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 14fd41b88..bd7c8d2f7 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -14,6 +14,34 @@ This project adheres to http://semver.org/[Semantic Versioning]. // `Fixed` for any bug fixes. // `Security` to invite users to upgrade in case of vulnerabilities. +v0.1.0 (unreleased) +------------------- + +Breaking changes +~~~~~~~~~~~~~~~~ + +- Support for legacy QtWebKit (before 5.212 which is distributed + independently from Qt) is dropped. +- Support for Python 3.4 is dropped. +- Support for Qt before 5.7 is dropped. +- New dependency on the QtSql module and Qt sqlite support. +- New dependency on ruamel.yaml; dropped PyYAML dependency. +- The QtWebEngine backend is now used by default if available. +- New config system which ignores the old config file. + +Major changes +~~~~~~~~~~~~~ + +- New completion engine based on sqlite, which allows to complete + the entire browsing history. +- Completely rewritten configuration system. + +v0.11.1 (unreleased) +-------------------- + +Fixes +~~~~~ + v0.11.0 ------- From a8120a23c4b24c203e07a8ca80105aa22c607e63 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 5 Jul 2017 22:13:24 +0200 Subject: [PATCH 180/337] Update comment for TabBarStyle --- qutebrowser/mainwindow/tabwidget.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 3dd0d28da..d1beea7c7 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -765,11 +765,9 @@ class TabBarStyle(QCommonStyle): # Otherwise, empty space will be shown after the last tab even # though the button width is set to 0 # - # In older PyQt-versions (5.2.1) QStyle does not have this - # attribute. + # QStyle.SE_TabBarScrollLeftButton was added in Qt 5.7 if sr == QStyle.SE_TabBarScrollLeftButton: return super().subElementRect(sr, opt, widget) - except AttributeError: pass From 0304040cbbf53043573a1b1104f4f0e9efd5ba02 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 5 Jul 2017 22:14:01 +0200 Subject: [PATCH 181/337] Update docs --- README.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.asciidoc b/README.asciidoc index daa008a83..98e13a538 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -193,6 +193,7 @@ Contributors, sorted by the number of commits in descending order: * knaggita * Oliver Caldwell * Nikolay Amiantov +* Marius * Julian Weigt * Tomasz Kramkowski * Sebastian Frysztak @@ -217,7 +218,6 @@ Contributors, sorted by the number of commits in descending order: * Michał Góral * Michael Ilsaas * Martin Zimmermann -* Marius * Link * Jussi Timperi * Cosmin Popescu From 6b4e0ad2bcd8d179baf8c2961651bdc469b3fae5 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 5 Jul 2017 22:14:49 +0200 Subject: [PATCH 182/337] Update changelog --- CHANGELOG.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index bd7c8d2f7..15e087c6b 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -42,6 +42,8 @@ v0.11.1 (unreleased) Fixes ~~~~~ +- Fixed empty space being shown after tabs in the tabbar in some cases. + v0.11.0 ------- From 3c9de92d5857b3cd26633cfd18c976678c07f789 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 6 Jul 2017 00:40:58 +0200 Subject: [PATCH 183/337] Add Gentoo instructions to backend warning --- qutebrowser/html/backend-warning.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qutebrowser/html/backend-warning.html b/qutebrowser/html/backend-warning.html index ffff0e59b..2b631d6a5 100644 --- a/qutebrowser/html/backend-warning.html +++ b/qutebrowser/html/backend-warning.html @@ -70,6 +70,8 @@ the qute://settings page or caret browsing). {{ install_webengine('qt5-qtwebengine') }} {% elif distribution.parsed == Distribution.opensuse %} {{ install_webengine('libqt5-qtwebengine') }} +{% elif distribution.parsed == Distribution.gentoo %} + {{ install_webengine('dev-qt/qtwebengine') }} {% else %} {{ unknown_system() }} {% endif %} From 57e4d4978b66fc7547e2aeafb2ddd1b708d19580 Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Thu, 6 Jul 2017 11:59:02 +0200 Subject: [PATCH 184/337] Use page title only for whitelisted extensions --- qutebrowser/browser/commands.py | 12 +++++++++++- .../data/downloads/download with no title.html | 8 ++++++++ tests/end2end/data/downloads/qutebrowser.png | Bin 0 -> 4278 bytes tests/end2end/features/downloads.feature | 14 ++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/end2end/data/downloads/download with no title.html create mode 100644 tests/end2end/data/downloads/qutebrowser.png diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index fbf3fe914..88499c412 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1443,12 +1443,22 @@ class CommandDispatcher: download_manager.get_mhtml(tab, target) else: qnam = tab.networkaccessmanager() + + # Downloads of URLs with file extensions in the whitelist will use + # the page title as the filename. + ext_whitelist = [".html", ".htm", ".php", ""] + _, ext = os.path.splitext(self._current_url().path()) + if ext.lower() in ext_whitelist and tab.title(): + suggested_fn = utils.sanitize_filename(tab.title()) + ext + else: + suggested_fn = None + download_manager.get( self._current_url(), user_agent=user_agent, qnam=qnam, target=target, - suggested_fn=utils.sanitize_filename(tab.title() + ".html") + suggested_fn=suggested_fn ) @cmdutils.register(instance='command-dispatcher', scope='window') diff --git a/tests/end2end/data/downloads/download with no title.html b/tests/end2end/data/downloads/download with no title.html new file mode 100644 index 000000000..da4352e59 --- /dev/null +++ b/tests/end2end/data/downloads/download with no title.html @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/tests/end2end/data/downloads/qutebrowser.png b/tests/end2end/data/downloads/qutebrowser.png new file mode 100644 index 0000000000000000000000000000000000000000..e8bbb6b5649a4300bbb55a60e79a84c3468d3ea8 GIT binary patch literal 4278 zcmV;n5J~TeP)rsbCgnS&R|uOlc#E%7;kgp?P=H1k&>}d zvLiQ1wov#Mlur|^B7LC69WVuK$dbdPop)+%%xiFXK4~E!Hc(R0D3~+Sz*jSEeonxq zMvJokj!048Z=o3g1#CSo$J9wSRoR&4GR*nK1WKX4ns zmhR#H@DD?c-iT2p1izjftGp|O=vBv7^hp6f708jfRmQ&arONQ$^s}gN_hyf zyg|_Hg1Tm?cff5!q$v;gkjD&NU70vUT$!Uk}QmU>j_V{XYeonn0(qA$xRN~YH z6^-5Z^S8i|5J^eBKnT#(7NZwn@mxr|!-==W%e0MvKoFm16zNG*VwD#?-4Y&sx@{+9 zP_-^kPl%QAC$1{k91!3Q3Q%W~RJBIbHhY*~x9Ri|C6LU~_<%0`Vb| z1_#tT1m%r_s`66uFXvKGR!nVGCCv?$R905ubm?7q`uTVh6&cF^{KqFLB;Va74V9bO8yU=u52V4prEGZ3p))sQ!L5L zJx5M6$TSUOv?oht<;*zJxE^-@aqr)Kcp)~~6o@tmJK2r2Iqv2u%)@-egR zV5)%VA4w~2WJLQqm(xjB`Z3au9ipMWwqHWF$EExt-uQ4QS?7Cg$j$KR;Y&t6@K4<} z=vDz6)8rdhs!3>HmzSH(fjwVPSKBY&qXX;?C+~0jGrRU3L)ZIsl$*k7h;<*ON&DI) z@r*mVTLq+~%RBOFd9Jl==sE}Yeo0pPu>lvbw7iNPds7FTZ!5z5BwmVKomMK}N{SYF zo#{&FitXE_T4H05-#rl@;?Vu0U^Ewlvm4L+qcUuj&>IX)DAT^*EACh zEnoiY&o@zU<>JpF>rF$I4Kj2ZLkRy@Gc`;!M$=z&6cqu^|DS5M!mBQ!hj0S`hu1sIZ*LO`H5 z>_2j%#{rh!pZv2--b+&n3i5MHvo+bNZ?vJR3W5H9cw4+MB>Z?)P}U&mZv9p^v7q(0 z29Q}SlcZschZe`=_)q6eg^G#@;qj#lOxp|urcJtq=hiM^h&2j;;<74^XP)Oo#yKur zDxj+7MpR5u1aUEu3>`U!VdJM089k_5zBUJzt!Z-kiX`!I%LkNQT1hmd^RBqom;K~zLA4UNrQxKzOBdyY_DYrb_ap3mx{0GBSqwP9g2 zohiYu^>ndV%#Sy_v&T52Tg1gg@XVUU%uE=|wUP>6{Ot!E&ouWpS`i)+#N$gBuwYJ7 z+g+m-i4#XNKRJoDFaMFsit=uGl$I1yUt2?9a7bI7!w}xu@GvR76gX2NaOfa3a|KjY zH+Up;D|~${tXMLSM;=JV&<#HLbT6OnK8(ZV>ank;syuw(ELJ~!4}QMgryG6!gScn$ zW29_&vwLPzQe2qdK>@msssRCmLt4^RHO-J37C^p9ST65sF?S@5>rr?i1b5v&gTH#qZEpg&#o?o+s__$~c!#q^x=LK+S7=Tm7JhD!$@4f3wru-KrMOi-8~TrPKU_0j}l6XrjM-O)2rm3;x_Ei<62Gr9|0b@rFK?q?Q3wZVY z6dqpnH2wiWXeu;c-_AvK0$tb1K6R2~hxU4$E4XdS7@l3bgt(Xpe#kh-I~#X#t)#-E zzU?t)aDXY@>FjvSsw%n$Q~+&ZqUeZFM#fuB?*tX)Wo&-$RT5_=6Kx%gx3@2Kb=4H+ zUn1x1DUT;^jSlD8)r+}3aU9o5D)`smZQzFt&qHvX;nTwj=CF5kt1XJOYd|hAwLM|> zjEOy7ikfTCLpKt zVqbF4?R>KP5KT>ezld~)(PJlZ_k9nNI5HG3FG)dRDIf1VFd)2Z>usj=V1FNC0wmsE zFf3FO;=>kA3V9aU(SipB`m=OlGMm2GHz0xmgoZ~k_r6~+bi}Oy>^^dqq%m;}?Rrz~ zGYX2jcZ2|~5(=|O?h?W3BD>PN* zcQ3BySHJzBM|;p6#!Q&X?7J5c5EP8l>EQTx`$#)>5M6KC>rUfF#<{<6xLlk(W$rJG zh_|-=mXMWb6%`tQqOf1ejtT$>3k~AEU;i_&Z~Th=M^5$_CI>(WMvNNEv{~~QGTBCM6{JT^A|H`{$gA%Ck}fv zhHl{F>xbqhDN|(R3fkoc6~UJ7 zIu~{FYc$OZKmR}if`Z*n9-!IQIAD{nZt9-4ZP!6|{@>9ac`Tee<3>T!mLzRcK%+%j z54gRpG~S#*jmO9-7o=WMD5$+*FYFE{Z*Sbi`V{k}4=`!W2olDPY%NyVyp;{Dg*F9T ze|!6`I|dVCCBA>d)Sm<@8zHAm;BdM4%Xew4e&Kb#*!z8thFMV*ez{^{duDI7e`nkw zzNNX8w_KX>B+w!98RQKKgCsv(=|0_bGm(>hilS@zRF)L-#| zQp2vi)1KZz0Tbgzbw;7trz2o6~XYtz0>)O9rsL?fbwGg6<@NU0n;t+8OLq6_~c1xrr(Br>DxQC)B zB+QsaP;gk!jeCLEs4(96&nI|x&Ej@n9;A?K-MtV4lXrp%aoUcIBK-{{|I)r*5ilx3 zl3Ca{gBUkt1at3uh{zag&y77qbVMl27TwAHcTMl|QboWY60F+J?)sW$Ii|!aFJu%M zVFFgQuOAX3@zLOPiJ-y5kb!?d5Xtu}V$6i8Em6#VA}S(;#0jIBJ97$2lSZQ`ZWnXP zWUKPB86Q&x2q9#8x?Cr$G72g0?I1vaWbOz_X@lT&vB1_%Fh)^T5~j~$X7YTuxq8jZ zLU>dRbv4yE9bL7lsv5CTVZ_CTV~q@Bc)XRV6GruzqZEieda+m~2AMyb_vlj`mQo8Y zGS&lY+|ip|aQIToQ?yoGzIcw4M-Oq=!iNZtjPYa`037yaq%;r+EIz*YSzz9%9{!Bf z2|}_>FHzQb@?73yzEVn&QDpo9$OnJ;I+wC zLB$+A>?A*+!+Ep1!Mc4q9dFX`#S~10{ zeWhzkBFGLeoNtDMG&n>O=213Yey^&^mKe8w?;}rd?a2yzr?AMf|nCwl?}c5>g~~Qi8C+D5NR=*BRw-i3pd0#buzm41@%{6lAckY?>KwoCA>~Bp0M2vn6V3qE+-hOWF(k YKhMHKf literal 0 HcmV?d00001 diff --git a/tests/end2end/features/downloads.feature b/tests/end2end/features/downloads.feature index 8a9e8c134..40333aee6 100644 --- a/tests/end2end/features/downloads.feature +++ b/tests/end2end/features/downloads.feature @@ -29,6 +29,20 @@ Feature: Downloading things from a website. And I wait until the download is finished Then the downloaded file Simple downloads.html should exist + Scenario: Using :download with no URL on an image + When I set storage -> prompt-download-directory to false + And I open data/downloads/qutebrowser.png + And I run :download + And I wait until the download is finished + Then the downloaded file qutebrowser.png should exist + + Scenario: Using :download with no URL and no page title + When I set storage -> prompt-download-directory to false + And I open data/downloads/download with no title.html + And I run :download + And I wait until the download is finished + Then the downloaded file download with no title.html should exist + Scenario: Using hints When I set storage -> prompt-download-directory to false And I open data/downloads/downloads.html From 94951d92a18f9535919398c703f41c982d201f4c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 6 Jul 2017 12:15:42 +0200 Subject: [PATCH 185/337] Simplify arg handling in test_version_output --- tests/unit/utils/test_version.py | 50 +++++++++++++++++++------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 32c3fcf5c..dae823ba9 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -799,18 +799,27 @@ def test_chromium_version_unpatched(qapp): assert version._chromium_version() not in ['', 'unknown', 'unavailable'] -@pytest.mark.parametrize(['git_commit', 'frozen', 'style', 'with_webkit', - 'known_distribution'], [ - (True, False, True, True, True), # normal - (False, False, True, True, True), # no git commit - (True, True, True, True, True), # frozen - (True, True, False, True, True), # no style - (True, False, True, False, True), # no webkit - (True, False, True, 'ng', True), # QtWebKit-NG - (True, False, True, True, False), # unknown Linux distribution +class VersionParams: + + def __init__(self, git_commit=True, frozen=False, style=True, + with_webkit=True, known_distribution=True): + self.git_commit = git_commit + self.frozen = frozen + self.style = style + self.with_webkit = with_webkit + self.known_distribution = known_distribution + + +@pytest.mark.parametrize('params', [ + VersionParams(), + VersionParams(git_commit=False), + VersionParams(frozen=True), + VersionParams(style=False), + VersionParams(with_webkit=False), + VersionParams(with_webkit='ng'), + VersionParams(known_distribution=False), ]) # pylint: disable=too-many-locals -def test_version_output(git_commit, frozen, style, with_webkit, - known_distribution, stubs, monkeypatch): +def test_version_output(params, stubs, monkeypatch): """Test version.version().""" class FakeWebEngineProfile: def httpUserAgent(self): @@ -820,7 +829,7 @@ def test_version_output(git_commit, frozen, style, with_webkit, patches = { 'qutebrowser.__file__': os.path.join(import_path, '__init__.py'), 'qutebrowser.__version__': 'VERSION', - '_git_str': lambda: ('GIT COMMIT' if git_commit else None), + '_git_str': lambda: ('GIT COMMIT' if params.git_commit else None), 'platform.python_implementation': lambda: 'PYTHON IMPLEMENTATION', 'platform.python_version': lambda: 'PYTHON VERSION', 'PYQT_VERSION_STR': 'PYQT VERSION', @@ -832,24 +841,25 @@ def test_version_output(git_commit, frozen, style, with_webkit, 'platform.architecture': lambda: ('ARCHITECTURE', ''), '_os_info': lambda: ['OS INFO 1', 'OS INFO 2'], '_path_info': lambda: {'PATH DESC': 'PATH NAME'}, - 'QApplication': (stubs.FakeQApplication(style='STYLE') if style else + 'QApplication': (stubs.FakeQApplication(style='STYLE') + if params.style else stubs.FakeQApplication(instance=None)), 'QLibraryInfo.location': (lambda _loc: 'QT PATH'), } substitutions = { - 'git_commit': '\nGit commit: GIT COMMIT' if git_commit else '', - 'style': '\nStyle: STYLE' if style else '', + 'git_commit': '\nGit commit: GIT COMMIT' if params.git_commit else '', + 'style': '\nStyle: STYLE' if params.style else '', 'qt': 'QT VERSION', - 'frozen': str(frozen), + 'frozen': str(params.frozen), 'import_path': import_path, } - if with_webkit: + if params.with_webkit: patches['qWebKitVersion'] = lambda: 'WEBKIT VERSION' patches['objects.backend'] = usertypes.Backend.QtWebKit patches['QWebEngineProfile'] = None - if with_webkit == 'ng': + if params.with_webkit == 'ng': backend = 'QtWebKit-NG' patches['qtutils.is_qtwebkit_ng'] = lambda: True else: @@ -862,7 +872,7 @@ def test_version_output(git_commit, frozen, style, with_webkit, patches['QWebEngineProfile'] = FakeWebEngineProfile substitutions['backend'] = 'QtWebEngine (Chromium CHROMIUMVERSION)' - if known_distribution: + if params.known_distribution: patches['distribution'] = lambda: version.DistributionInfo( parsed=version.Distribution.arch, version=None, pretty='LINUX DISTRIBUTION', id='arch') @@ -877,7 +887,7 @@ def test_version_output(git_commit, frozen, style, with_webkit, for attr, val in patches.items(): monkeypatch.setattr('qutebrowser.utils.version.' + attr, val) - if frozen: + if params.frozen: monkeypatch.setattr(sys, 'frozen', True, raising=False) else: monkeypatch.delattr(sys, 'frozen', raising=False) From 911e59b0f49cbc1594dfaf608ff93f6acd2ca6a4 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 6 Jul 2017 12:23:08 +0200 Subject: [PATCH 186/337] Improve version output without SSL support --- qutebrowser/utils/version.py | 7 +++---- tests/unit/utils/test_version.py | 16 +++++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index e1cc26e64..dd36f381c 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -326,11 +326,10 @@ def version(): lines += _module_versions() - lines += ['pdf.js: {}'.format(_pdfjs_version())] - lines += [ - 'SSL: {}'.format(QSslSocket.sslLibraryVersionString()), - '', + 'pdf.js: {}'.format(_pdfjs_version()), + 'QtNetwork SSL: {}\n'.format(QSslSocket.sslLibraryVersionString() + if QSslSocket.supportsSsl() else 'no'), ] qapp = QApplication.instance() diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index dae823ba9..aede735d4 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -759,14 +759,16 @@ class FakeQSslSocket: Attributes: _version: What QSslSocket::sslLibraryVersionString() should return. + _support: Whether SSL is supported. """ - def __init__(self, version=None): + def __init__(self, version=None, support=True): self._version = version + self._support = support def supportsSsl(self): """Fake for QSslSocket::supportsSsl().""" - return True + return self._support def sslLibraryVersionString(self): """Fake for QSslSocket::sslLibraryVersionString().""" @@ -802,12 +804,13 @@ def test_chromium_version_unpatched(qapp): class VersionParams: def __init__(self, git_commit=True, frozen=False, style=True, - with_webkit=True, known_distribution=True): + with_webkit=True, known_distribution=True, ssl_support=True): self.git_commit = git_commit self.frozen = frozen self.style = style self.with_webkit = with_webkit self.known_distribution = known_distribution + self.ssl_support = ssl_support @pytest.mark.parametrize('params', [ @@ -818,6 +821,7 @@ class VersionParams: VersionParams(with_webkit=False), VersionParams(with_webkit='ng'), VersionParams(known_distribution=False), + VersionParams(ssl_support=False), ]) # pylint: disable=too-many-locals def test_version_output(params, stubs, monkeypatch): """Test version.version().""" @@ -836,7 +840,7 @@ def test_version_output(params, stubs, monkeypatch): 'earlyinit.qt_version': lambda: 'QT VERSION', '_module_versions': lambda: ['MODULE VERSION 1', 'MODULE VERSION 2'], '_pdfjs_version': lambda: 'PDFJS VERSION', - 'QSslSocket': FakeQSslSocket('SSL VERSION'), + 'QSslSocket': FakeQSslSocket('SSL VERSION', params.ssl_support), 'platform.platform': lambda: 'PLATFORM', 'platform.architecture': lambda: ('ARCHITECTURE', ''), '_os_info': lambda: ['OS INFO 1', 'OS INFO 2'], @@ -884,6 +888,8 @@ def test_version_output(params, stubs, monkeypatch): substitutions['linuxdist'] = '' substitutions['osinfo'] = 'OS INFO 1\nOS INFO 2\n' + substitutions['ssl'] = 'SSL VERSION' if params.ssl_support else 'no' + for attr, val in patches.items(): monkeypatch.setattr('qutebrowser.utils.version.' + attr, val) @@ -903,7 +909,7 @@ def test_version_output(params, stubs, monkeypatch): MODULE VERSION 1 MODULE VERSION 2 pdf.js: PDFJS VERSION - SSL: SSL VERSION + QtNetwork SSL: {ssl} {style} Platform: PLATFORM, ARCHITECTURE{linuxdist} Frozen: {frozen} From 66168a5b49e224df019d65e3a51b3dda9b1aede3 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 6 Jul 2017 12:25:11 +0200 Subject: [PATCH 187/337] Add test ids to test_version_output --- tests/unit/utils/test_version.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index aede735d4..c60150143 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -803,8 +803,9 @@ def test_chromium_version_unpatched(qapp): class VersionParams: - def __init__(self, git_commit=True, frozen=False, style=True, + def __init__(self, name, git_commit=True, frozen=False, style=True, with_webkit=True, known_distribution=True, ssl_support=True): + self.name = name self.git_commit = git_commit self.frozen = frozen self.style = style @@ -814,15 +815,15 @@ class VersionParams: @pytest.mark.parametrize('params', [ - VersionParams(), - VersionParams(git_commit=False), - VersionParams(frozen=True), - VersionParams(style=False), - VersionParams(with_webkit=False), - VersionParams(with_webkit='ng'), - VersionParams(known_distribution=False), - VersionParams(ssl_support=False), -]) # pylint: disable=too-many-locals + VersionParams('normal'), + VersionParams('no-git-commit', git_commit=False), + VersionParams('frozen', frozen=True), + VersionParams('no-style', style=False), + VersionParams('no-webkit', with_webkit=False), + VersionParams('webkit-ng', with_webkit='ng'), + VersionParams('unknown-dist', known_distribution=False), + VersionParams('no-ssl', ssl_support=False), +], ids=lambda param: param.name) # pylint: disable=too-many-locals def test_version_output(params, stubs, monkeypatch): """Test version.version().""" class FakeWebEngineProfile: From 338d62204e11558976530a3003ff74c533601904 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 6 Jul 2017 12:36:11 +0200 Subject: [PATCH 188/337] Make TestModuleVersions in test_version more maintainable --- tests/unit/utils/test_version.py | 87 ++++++++++++++++---------------- 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index c60150143..5623d0ce3 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -21,6 +21,7 @@ import io import sys +import collections import os.path import subprocess import contextlib @@ -475,29 +476,32 @@ class ImportFake: """A fake for __import__ which is used by the import_fake fixture. Attributes: - exists: A dict mapping module names to bools. If True, the import will - success. Otherwise, it'll fail with ImportError. + modules: A dict mapping module names to bools. If True, the import will + success. Otherwise, it'll fail with ImportError. version_attribute: The name to use in the fake modules for the version attribute. version: The version to use for the modules. _real_import: Saving the real __import__ builtin so the imports can be - done normally for modules not in self.exists. + done normally for modules not in self. modules. """ def __init__(self): - self.exists = { - 'sip': True, - 'colorama': True, - 'pypeg2': True, - 'jinja2': True, - 'pygments': True, - 'yaml': True, - 'cssutils': True, - 'typing': True, - 'PyQt5.QtWebEngineWidgets': True, - 'PyQt5.QtWebKitWidgets': True, - 'OpenGL': True, - } + self.modules = collections.OrderedDict([ + ('sip', True), + ('colorama', True), + ('pypeg2', True), + ('jinja2', True), + ('pygments', True), + ('yaml', True), + ('cssutils', True), + ('typing', True), + ('OpenGL', True), + ('PyQt5.QtWebEngineWidgets', True), + ('PyQt5.QtWebKitWidgets', True), + ]) + self.no_version_attribute = ['sip', 'typing', + 'PyQt5.QtWebEngineWidgets', + 'PyQt5.QtWebKitWidgets'] self.version_attribute = '__version__' self.version = '1.2.3' self._real_import = builtins.__import__ @@ -509,10 +513,10 @@ class ImportFake: The imported fake module, or None if normal importing should be used. """ - if name not in self.exists: + if name not in self.modules: # Not one of the modules to test -> use real import return None - elif self.exists[name]: + elif self.modules[name]: ns = types.SimpleNamespace() if self.version_attribute is not None: setattr(ns, self.version_attribute, self.version) @@ -551,14 +555,14 @@ class TestModuleVersions: """Tests for _module_versions().""" - @pytest.mark.usefixtures('import_fake') - def test_all_present(self): + def test_all_present(self, import_fake): """Test with all modules present in version 1.2.3.""" - expected = ['sip: yes', 'colorama: 1.2.3', 'pypeg2: 1.2.3', - 'jinja2: 1.2.3', 'pygments: 1.2.3', 'yaml: 1.2.3', - 'cssutils: 1.2.3', 'typing: yes', 'OpenGL: 1.2.3', - 'PyQt5.QtWebEngineWidgets: yes', - 'PyQt5.QtWebKitWidgets: yes'] + expected = [] + for name in import_fake.modules: + if name in import_fake.no_version_attribute: + expected.append('{}: yes'.format(name)) + else: + expected.append('{}: 1.2.3'.format(name)) assert version._module_versions() == expected @pytest.mark.parametrize('module, idx, expected', [ @@ -574,36 +578,31 @@ class TestModuleVersions: idx: The index where the given text is expected. expected: The expected text. """ - import_fake.exists[module] = False + import_fake.modules[module] = False assert version._module_versions()[idx] == expected - @pytest.mark.parametrize('value, expected', [ - ('VERSION', ['sip: yes', 'colorama: 1.2.3', 'pypeg2: yes', - 'jinja2: yes', 'pygments: yes', 'yaml: yes', - 'cssutils: yes', 'typing: yes', 'OpenGL: yes', - 'PyQt5.QtWebEngineWidgets: yes', - 'PyQt5.QtWebKitWidgets: yes']), - ('SIP_VERSION_STR', ['sip: 1.2.3', 'colorama: yes', 'pypeg2: yes', - 'jinja2: yes', 'pygments: yes', 'yaml: yes', - 'cssutils: yes', 'typing: yes', 'OpenGL: yes', - 'PyQt5.QtWebEngineWidgets: yes', - 'PyQt5.QtWebKitWidgets: yes']), - (None, ['sip: yes', 'colorama: yes', 'pypeg2: yes', 'jinja2: yes', - 'pygments: yes', 'yaml: yes', 'cssutils: yes', 'typing: yes', - 'OpenGL: yes', 'PyQt5.QtWebEngineWidgets: yes', - 'PyQt5.QtWebKitWidgets: yes']), + @pytest.mark.parametrize('attribute, expected_modules', [ + ('VERSION', ['colorama']), + ('SIP_VERSION_STR', ['sip']), + (None, []), ]) - def test_version_attribute(self, value, expected, import_fake): + def test_version_attribute(self, attribute, expected_modules, import_fake): """Test with a different version attribute. VERSION is tested for old colorama versions, and None to make sure things still work if some package suddenly doesn't have __version__. Args: - value: The name of the version attribute. + attribute: The name of the version attribute. expected: The expected return value. """ - import_fake.version_attribute = value + import_fake.version_attribute = attribute + expected = [] + for name in import_fake.modules: + if name in expected_modules: + expected.append('{}: 1.2.3'.format(name)) + else: + expected.append('{}: yes'.format(name)) assert version._module_versions() == expected @pytest.mark.parametrize('name, has_version', [ From cee0aa3adc01faed0f18e982fb3a37e04c8a469f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 6 Jul 2017 07:36:59 -0400 Subject: [PATCH 189/337] Show error dialog is sql isn't available. If creating the sql database fails, show an error dialog assuming sqlite is not installed. This removes the isDriverAvailable check as it was true even with sqlite uninstalled. sql.version now inits itself if sql is not already initialized and prints 'UNAVAILABLE ()' if init fails. This is to avoid cascading errors, where one error would create a crash dialog, which calls sql.version, which would create another error. --- qutebrowser/app.py | 8 ++++++-- qutebrowser/misc/earlyinit.py | 7 ------- qutebrowser/misc/sql.py | 14 +++++++++++--- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 677f79a7b..46c7578bd 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -85,7 +85,6 @@ def run(args): if args.version: # we need to init sql to print the sql version # we can use an in-memory database as we just want to query the version - sql.init(':memory:') print(version.version()) sys.exit(usertypes.Exit.ok) @@ -429,7 +428,12 @@ def _init_modules(args, crash_handler): keyconf.init(qApp) log.init.debug("Initializing sql...") - sql.init(os.path.join(standarddir.data(), 'history.sqlite')) + try: + sql.init(os.path.join(standarddir.data(), 'history.sqlite')) + except sql.SqlException as e: + error.handle_fatal_exc(e, args, 'Is sqlite installed?', + pre_text='Failed to initialize SQL') + sys.exit(usertypes.Exit.err_init) log.init.debug("Initializing web history...") history.init(qApp) diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index 68f5db71e..fb09b4ab2 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -396,12 +396,6 @@ def check_optimize_flag(): "unexpected behavior may occur.") -def check_sqlite(): - from PyQt5.QtSql import QSqlDatabase - if not QSqlDatabase.isDriverAvailable('QSQLITE'): - _die('sqlite driver not available! Is sqlite installed?') - - def set_backend(backend): """Set the objects.backend global to the given backend (as string).""" from qutebrowser.misc import objects @@ -441,5 +435,4 @@ def earlyinit(args): check_libraries(backend) check_ssl_support(backend) check_optimize_flag() - check_sqlite() set_backend(backend) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 35c5359dd..4ae37f01f 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -37,6 +37,8 @@ class SqlException(Exception): def init(db_path): """Initialize the SQL database connection.""" database = QSqlDatabase.addDatabase('QSQLITE') + if not database.isValid(): + raise SqlException('Failed to add database. Is sqlite installed?') database.setDatabaseName(db_path) if not database.open(): raise SqlException("Failed to open sqlite database at {}: {}" @@ -50,9 +52,15 @@ def close(): def version(): """Return the sqlite version string.""" - q = Query("select sqlite_version()") - q.run() - return q.value() + try: + if not QSqlDatabase.database().isOpen(): + init(':memory:') + ver = Query("select sqlite_version()").run().value() + close() + return ver + return Query("select sqlite_version()").run().value() + except SqlException as e: + return 'UNAVAILABLE ({})'.format(e) class Query(QSqlQuery): From 1dd5f06a4fa6e7b57af10ed3deeda7c4db119896 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 6 Jul 2017 08:02:16 -0400 Subject: [PATCH 190/337] Fix debug-dump-history behavior. Ensure the file is closed before printing the success message. This will hopefully fix the AppVeyor tests. --- qutebrowser/browser/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index 86c6a6cd9..e583fba40 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -282,7 +282,7 @@ class WebHistory(sql.SqlTable): try: with open(dest, 'w', encoding='utf-8') as f: f.write('\n'.join(lines)) - message.info("Dumped history to {}".format(dest)) + message.info("Dumped history to {}".format(dest)) except OSError as e: raise cmdexc.CommandError('Could not write history: {}', e) From d179450c29a9dfbe0eff08ca84fddfbd3967615d Mon Sep 17 00:00:00 2001 From: Christian Helbling Date: Thu, 6 Jul 2017 15:35:52 +0200 Subject: [PATCH 191/337] :fullscreen, enter video fullscreen, :fullscreen, exit video fullscreen should not go into video fullscreen --- qutebrowser/mainwindow/mainwindow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index 6edabda09..88ffcf383 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -488,7 +488,7 @@ class MainWindow(QWidget): if on: self.state_before_fullscreen = self.windowState() self.showFullScreen() - else: + elif self.isFullScreen(): self.setWindowState(self.state_before_fullscreen) log.misc.debug('on: {}, state before fullscreen: {}'.format( on, debug.qflags_key(Qt, self.state_before_fullscreen))) From 3bfafb5e5066b40dbe41cffe1e0fc3d27e504cfd Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Thu, 6 Jul 2017 17:41:54 +0200 Subject: [PATCH 192/337] Refactor suggested_fn_from_title, add unit tests --- qutebrowser/browser/commands.py | 11 +++------- qutebrowser/browser/downloads.py | 20 +++++++++++++++++ tests/end2end/features/downloads.feature | 7 ------ tests/unit/browser/webkit/test_downloads.py | 24 +++++++++++++++++++++ 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 88499c412..3e918a32a 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1444,14 +1444,9 @@ class CommandDispatcher: else: qnam = tab.networkaccessmanager() - # Downloads of URLs with file extensions in the whitelist will use - # the page title as the filename. - ext_whitelist = [".html", ".htm", ".php", ""] - _, ext = os.path.splitext(self._current_url().path()) - if ext.lower() in ext_whitelist and tab.title(): - suggested_fn = utils.sanitize_filename(tab.title()) + ext - else: - suggested_fn = None + suggested_fn = downloads.suggested_fn_from_title( + self._current_url().path(), tab.title() + ) download_manager.get( self._current_url(), diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py index c7cb5ad8e..35b3d26c6 100644 --- a/qutebrowser/browser/downloads.py +++ b/qutebrowser/browser/downloads.py @@ -182,6 +182,26 @@ def transform_path(path): return path +def suggested_fn_from_title(url, title=None): + """Suggest a filename depending on the URL extension and page title. + Args: + url: a string with the URL path + title: the page title string + + Returns None if the extension is not in the whitelist + or if there is no page title. + """ + ext_whitelist = [".html", ".htm", ".php", ""] + _, ext = os.path.splitext(url) + if ext.lower() in ext_whitelist and title: + suggested_fn = utils.sanitize_filename(title) + if not suggested_fn.endswith(ext): + suggested_fn += ext + else: + suggested_fn = None + return suggested_fn + + class NoFilenameError(Exception): """Raised when we can't find out a filename in DownloadTarget.""" diff --git a/tests/end2end/features/downloads.feature b/tests/end2end/features/downloads.feature index 40333aee6..f636b3247 100644 --- a/tests/end2end/features/downloads.feature +++ b/tests/end2end/features/downloads.feature @@ -36,13 +36,6 @@ Feature: Downloading things from a website. And I wait until the download is finished Then the downloaded file qutebrowser.png should exist - Scenario: Using :download with no URL and no page title - When I set storage -> prompt-download-directory to false - And I open data/downloads/download with no title.html - And I run :download - And I wait until the download is finished - Then the downloaded file download with no title.html should exist - Scenario: Using hints When I set storage -> prompt-download-directory to false And I open data/downloads/downloads.html diff --git a/tests/unit/browser/webkit/test_downloads.py b/tests/unit/browser/webkit/test_downloads.py index 72b533cc7..d22338717 100644 --- a/tests/unit/browser/webkit/test_downloads.py +++ b/tests/unit/browser/webkit/test_downloads.py @@ -30,6 +30,30 @@ def test_download_model(qapp, qtmodeltester, config_stub, cookiejar_and_cache): qtmodeltester.check(model) +@pytest.mark.parametrize('url, title, out', [ + ('https://qutebrowser.org/img/cheatsheet-big.png', + 'cheatsheet-big.png (3342×2060)', + None), + ('http://qutebrowser.org/INSTALL.html', + 'Installing qutebrowser | qutebrowser', + 'Installing qutebrowser _ qutebrowser.html'), + ('http://qutebrowser.org/INSTALL.html.html', + 'Installing qutebrowser | qutebrowser', + 'Installing qutebrowser _ qutebrowser.html'), + ('http://qutebrowser.org/', + 'qutebrowser | qutebrowser', + 'qutebrowser _ qutebrowser'), + ('https://github.com/qutebrowser/qutebrowser/releases', + 'Releases · qutebrowser/qutebrowser', + 'Releases · qutebrowser_qutebrowser'), + ('http://qutebrowser.org/page-with-no-title.html', + '', + None), +]) +def test_page_titles(url, title, out): + assert downloads.suggested_fn_from_title(url, title) == out + + class TestDownloadTarget: def test_base(self): From 82d194cf2eb1462b385a552c2ac2d48e0aa07918 Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Thu, 6 Jul 2017 21:37:11 +0200 Subject: [PATCH 193/337] Improve function docstring, add more tests --- qutebrowser/browser/downloads.py | 14 ++++++++------ tests/unit/browser/webkit/test_downloads.py | 16 +++++++++++----- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py index 35b3d26c6..e9499600e 100644 --- a/qutebrowser/browser/downloads.py +++ b/qutebrowser/browser/downloads.py @@ -182,20 +182,22 @@ def transform_path(path): return path -def suggested_fn_from_title(url, title=None): +def suggested_fn_from_title(url_path, title=None): """Suggest a filename depending on the URL extension and page title. + Args: - url: a string with the URL path + url_path: a string with the URL path title: the page title string - Returns None if the extension is not in the whitelist - or if there is no page title. + Return: + The download filename based on the title, or None if the extension is + not found in the whitelist (or if there is no page title). """ ext_whitelist = [".html", ".htm", ".php", ""] - _, ext = os.path.splitext(url) + _, ext = os.path.splitext(url_path) if ext.lower() in ext_whitelist and title: suggested_fn = utils.sanitize_filename(title) - if not suggested_fn.endswith(ext): + if not suggested_fn.lower().endswith(ext.lower()): suggested_fn += ext else: suggested_fn = None diff --git a/tests/unit/browser/webkit/test_downloads.py b/tests/unit/browser/webkit/test_downloads.py index d22338717..c9451b949 100644 --- a/tests/unit/browser/webkit/test_downloads.py +++ b/tests/unit/browser/webkit/test_downloads.py @@ -31,21 +31,27 @@ def test_download_model(qapp, qtmodeltester, config_stub, cookiejar_and_cache): @pytest.mark.parametrize('url, title, out', [ - ('https://qutebrowser.org/img/cheatsheet-big.png', - 'cheatsheet-big.png (3342×2060)', - None), ('http://qutebrowser.org/INSTALL.html', 'Installing qutebrowser | qutebrowser', 'Installing qutebrowser _ qutebrowser.html'), - ('http://qutebrowser.org/INSTALL.html.html', - 'Installing qutebrowser | qutebrowser', + ('http://qutebrowser.org/INSTALL.html', + 'Installing qutebrowser | qutebrowser.html', 'Installing qutebrowser _ qutebrowser.html'), + ('http://qutebrowser.org/INSTALL.HTML', + 'Installing qutebrowser | qutebrowser', + 'Installing qutebrowser _ qutebrowser.HTML'), + ('http://qutebrowser.org/INSTALL.html', + 'Installing qutebrowser | qutebrowser.HTML', + 'Installing qutebrowser _ qutebrowser.HTML'), ('http://qutebrowser.org/', 'qutebrowser | qutebrowser', 'qutebrowser _ qutebrowser'), ('https://github.com/qutebrowser/qutebrowser/releases', 'Releases · qutebrowser/qutebrowser', 'Releases · qutebrowser_qutebrowser'), + ('https://qutebrowser.org/img/cheatsheet-big.png', + 'cheatsheet-big.png (3342×2060)', + None), ('http://qutebrowser.org/page-with-no-title.html', '', None), From 6a8d2ac826e4f4a02bc183a17aee497a7773aec7 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 6 Jul 2017 23:18:29 +0200 Subject: [PATCH 194/337] Disable search workaround for Qt 5.9.2 --- qutebrowser/browser/webengine/webenginetab.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 0a4c2dfc7..79688c870 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -715,7 +715,8 @@ class WebEngineTab(browsertab.AbstractTab): @pyqtSlot() def _on_load_started(self): """Clear search when a new load is started if needed.""" - if qtutils.version_check('5.9'): + if (qtutils.version_check('5.9') and + not qtutils.version_check('5.9.2')): # WORKAROUND for # https://bugreports.qt.io/browse/QTBUG-61506 self.search.clear() From 6d9e5dc931bff543eebaf97bb063b7c7c24a0224 Mon Sep 17 00:00:00 2001 From: Christian Helbling Date: Fri, 7 Jul 2017 11:30:18 +0200 Subject: [PATCH 195/337] avoid too long lines --- qutebrowser/browser/commands.py | 7 ++++--- qutebrowser/mainwindow/mainwindow.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 9f7cef9d4..f0585bba0 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -2171,9 +2171,10 @@ class CommandDispatcher: window = self._tabbed_browser.window() if window.isFullScreen(): - window.setWindowState(window.state_before_fullscreen & ~Qt.WindowFullScreen) + window.setWindowState( + window.state_before_fullscreen & ~Qt.WindowFullScreen) else: window.state_before_fullscreen = window.windowState() window.showFullScreen() - log.misc.debug('state before fullscreen: {}' - .format(debug.qflags_key(Qt, window.state_before_fullscreen))) + log.misc.debug('state before fullscreen: {}'.format( + debug.qflags_key(Qt, window.state_before_fullscreen))) diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index 88ffcf383..2f43ba58e 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -30,7 +30,8 @@ from PyQt5.QtWidgets import QWidget, QVBoxLayout, QApplication, QSizePolicy from qutebrowser.commands import runners, cmdutils from qutebrowser.config import config -from qutebrowser.utils import message, log, usertypes, qtutils, objreg, utils, debug +from qutebrowser.utils import (message, log, usertypes, qtutils, objreg, utils, + debug) from qutebrowser.mainwindow import tabbedbrowser, messageview, prompt from qutebrowser.mainwindow.statusbar import bar from qutebrowser.completion import completionwidget, completer From 5098aa388b9802c5279322f2500ec19da439bc20 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 7 Jul 2017 14:28:36 +0200 Subject: [PATCH 196/337] build_release: Fail GitHub uploads early --- scripts/dev/build_release.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index e6a4f89e1..6a6adb5bd 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -292,6 +292,13 @@ def build_sdist(): return artifacts +def read_github_token(): + """Read the GitHub API token from disk.""" + token_file = os.path.join(os.path.expanduser('~'), '.gh_token') + with open(token_file, encoding='ascii') as f: + token = f.read().strip() + + def github_upload(artifacts, tag): """Upload the given artifacts to GitHub. @@ -302,9 +309,7 @@ def github_upload(artifacts, tag): import github3 utils.print_title("Uploading to github...") - token_file = os.path.join(os.path.expanduser('~'), '.gh_token') - with open(token_file, encoding='ascii') as f: - token = f.read().strip() + token = read_github_token() gh = github3.login(token=token) repo = gh.repository('qutebrowser', 'qutebrowser') @@ -341,6 +346,12 @@ def main(): upload_to_pypi = False + if args.upload is not None: + # Fail early when trying to upload without github3 installed + # or without API token + import github3 + read_github_token() + if os.name == 'nt': if sys.maxsize > 2**32: # WORKAROUND From af6d833c50df281da4b4536d2f1b1215740fa3a7 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 7 Jul 2017 15:18:05 +0200 Subject: [PATCH 197/337] Fix build_release.py --- scripts/dev/build_release.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 6a6adb5bd..bb5733cad 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -297,6 +297,7 @@ def read_github_token(): token_file = os.path.join(os.path.expanduser('~'), '.gh_token') with open(token_file, encoding='ascii') as f: token = f.read().strip() + return token def github_upload(artifacts, tag): @@ -349,7 +350,7 @@ def main(): if args.upload is not None: # Fail early when trying to upload without github3 installed # or without API token - import github3 + import github3 # pylint: disable=unused-variable read_github_token() if os.name == 'nt': From 215503ba5901968573562306dbecfc47ef4a85a4 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 7 Jul 2017 15:21:18 +0200 Subject: [PATCH 198/337] Remove now useless suppression --- tests/unit/utils/test_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 5623d0ce3..6717cf75b 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -822,7 +822,7 @@ class VersionParams: VersionParams('webkit-ng', with_webkit='ng'), VersionParams('unknown-dist', known_distribution=False), VersionParams('no-ssl', ssl_support=False), -], ids=lambda param: param.name) # pylint: disable=too-many-locals +], ids=lambda param: param.name) def test_version_output(params, stubs, monkeypatch): """Test version.version().""" class FakeWebEngineProfile: From f7dbd3c283f9564d5d19023a0876daf8cedcc217 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 7 Jul 2017 17:58:38 +0200 Subject: [PATCH 199/337] Add initial CODEOWNERS file --- .github/CODEOWNERS | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..f623e0a93 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,10 @@ +* @The-Compiler + +qutebrowser/browser/history.py @The-Compiler @rcorre +qutebrowser/completion/* @The-Compiler @rcorre +qutebrowser/misc/sql.py @The-Compiler @rcorre +tests/end2end/features/completion.feature @The-Compiler @rcorre +tests/end2end/features/test_completion_bdd.py @The-Compiler @rcorre +tests/unit/browser/test_history.py @The-Compiler @rcorre +tests/unit/completion/* @The-Compiler @rcorre +tests/unit/misc/test_sql.py @The-Compiler @rcorre From a572b0f34d3c35d1c584c5fe7926d5b29c52d876 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 7 Jul 2017 18:40:57 +0200 Subject: [PATCH 200/337] Update docs --- CHANGELOG.asciidoc | 6 ++++++ README.asciidoc | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 15e087c6b..8a41c625e 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -36,6 +36,12 @@ Major changes the entire browsing history. - Completely rewritten configuration system. +Fixes +~~~~~ + +- Exiting fullscreen via `:fullscreen` or buttons on a page now + restores the correct previous window state (maximized/fullscreen). + v0.11.1 (unreleased) -------------------- diff --git a/README.asciidoc b/README.asciidoc index fff189cee..3ed3bc054 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -163,9 +163,9 @@ Contributors, sorted by the number of commits in descending order: * Kevin Velghe * Raphael Pierzina * Joel Torstensson +* Jay Kamat * Patric Schmitz * Tarcisio Fedrizzi -* Jay Kamat * Claude * Philipp Hansch * Fritz Reichwald @@ -211,6 +211,7 @@ Contributors, sorted by the number of commits in descending order: * Halfwit * David Vogt * Claire Cavanaugh +* Christian Helbling * rikn00 * kanikaa1234 * haitaka From 0de0bbfa712d7b537b8be030b051f9d2b6b57c6c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sat, 8 Jul 2017 10:46:08 +0200 Subject: [PATCH 201/337] Fix :restart with private browsing mode --- CHANGELOG.asciidoc | 1 + qutebrowser/app.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 8a41c625e..d8e21c629 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -49,6 +49,7 @@ Fixes ~~~~~ - Fixed empty space being shown after tabs in the tabbar in some cases. +- Fixed `:restart` in private browsing mode. v0.11.0 ------- diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 65e8d8181..e0873195e 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -634,7 +634,7 @@ class Quitter: # Save the session if one is given. if session is not None: session_manager = objreg.get('session-manager') - session_manager.save(session) + session_manager.save(session, with_private=True) # Open a new process and immediately shutdown the existing one try: args, cwd = self._get_restart_args(pages, session) From ad615941a28e0a382c93bebc2852c1812ce12c07 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sat, 8 Jul 2017 11:12:43 +0200 Subject: [PATCH 202/337] Replace OS X with macOS --- CHANGELOG.asciidoc | 18 +++++++++--------- CONTRIBUTING.asciidoc | 2 +- INSTALL.asciidoc | 6 +++--- doc/quickstart.asciidoc | 2 +- doc/userscripts.asciidoc | 2 +- pytest.ini | 6 +++--- qutebrowser/mainwindow/tabwidget.py | 2 +- qutebrowser/misc/ipc.py | 8 ++++---- qutebrowser/utils/standarddir.py | 2 +- qutebrowser/utils/utils.py | 2 +- scripts/dev/build_release.py | 12 ++++++------ tests/conftest.py | 4 ++-- tests/end2end/conftest.py | 2 +- tests/end2end/features/conftest.py | 2 +- tests/end2end/features/hints.feature | 2 +- tests/end2end/features/misc.feature | 2 +- tests/end2end/features/prompts.feature | 4 ++-- tests/end2end/features/yankpaste.feature | 2 +- tests/end2end/fixtures/testprocess.py | 2 +- tests/unit/misc/test_ipc.py | 14 +++++++------- tests/unit/utils/test_qtutils.py | 2 +- tests/unit/utils/test_standarddir.py | 4 ++-- tests/unit/utils/test_version.py | 10 +++++----- 23 files changed, 56 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d8e21c629..897afabf7 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -220,7 +220,7 @@ Added - Open tabs are now auto-saved on each successful load and restored in case of a crash - `:jseval` now has a `--file` flag so you can pass a javascript file - `:session-save` now has a `--only-active-window` flag to only save the active window -- OS X builds are back, and built with QtWebEngine +- macOS builds are back, and built with QtWebEngine Changed ~~~~~~~ @@ -522,7 +522,7 @@ Fixed - Fix crash when pressing enter without a command - Adjust error message to point out QtWebEngine is unsupported with the OS X .app currently. -- Hide Harfbuzz warning with the OS X .app +- Hide Harfbuzz warning with the macOS .app v0.8.0 ------ @@ -885,7 +885,7 @@ Fixed - Fixed scrolling to the very left/right with `:scroll-perc`. - Using an external editor should now work correctly with some funny chars (U+2028/U+2029/BOM). -- Movements in caret mode now should work correctly on OS X and Windows. +- Movements in caret mode now should work correctly on macOS and Windows. - Fixed upgrade from earlier config versions. - Fixed crash when killing a running userscript. - Fixed characters being passed through when shifted with @@ -960,7 +960,7 @@ Changed - The completion widget doesn't show a border anymore. - The tabbar doesn't display ugly arrows anymore if there isn't enough space for all tabs. -- Some insignificant Qt warnings which were printed on OS X are now hidden. +- Some insignificant Qt warnings which were printed on macOS are now hidden. - Better support for Qt 5.5 and Python 3.5. Fixed @@ -1071,7 +1071,7 @@ Fixed - Fixed AssertionError when closing many windows quickly. - Various fixes for deprecated key bindings and auto-migrations. - Workaround for qutebrowser not starting when there are NUL-bytes in the history (because of a currently unknown bug). -- Fixed handling of keybindings containing Ctrl/Meta on OS X. +- Fixed handling of keybindings containing Ctrl/Meta on macOS. - Fixed crash when downloading a URL without filename (e.g. magnet links) via "Save as...". - Fixed exception when starting qutebrowser with `:set` as argument. - Fixed horrible completion performance when the `shrink` option was set. @@ -1169,7 +1169,7 @@ Changed - Add a `:search` command in addition to `/foo` so it's more visible and can be used from scripts. - Various improvements to documentation, logging, and the crash reporter. - Expand `~` to the users home directory with `:run-userscript`. -- Improve the userscript runner on Linux/OS X by using `QSocketNotifier`. +- Improve the userscript runner on Linux/macOS by using `QSocketNotifier`. - Add luakit-like `gt`/`gT` keybindings to cycle through tabs. - Show default value for config values in the completion. - Clone tab icon, tab text and zoom level when cloning tabs. @@ -1189,7 +1189,7 @@ Changed * `init_venv.py` and `run_checks.py` have been replaced by http://tox.readthedocs.org/[tox]. Install tox and run `tox -e mkvenv` instead. * The tests now use http://pytest.org/[pytest] * Many new tests added - * Mac Mini buildbot to run the tests on OS X. + * Mac Mini buildbot to run the tests on macOS. * Coverage recording via http://nedbatchelder.com/code/coverage/[coverage.py]. * New `--pdb-postmortem argument` to drop into the pdb debugger on exceptions. * Use https://github.com/ionelmc/python-hunter[hunter] for line tracing instead of a selfmade solution. @@ -1325,7 +1325,7 @@ Fixed * Fix rare exception when a key is pressed shortly after opening a window * Fix exception with certain invalid URLs like `http:foo:0` -* Work around Qt bug which renders checkboxes on OS X unusable +* Work around Qt bug which renders checkboxes on macOS unusable * Fix exception when a local files can't be read in `:adblock-update` * Hide 2 more Qt warnings. * Add `!important` to hint CSS so websites don't override the hint look @@ -1361,7 +1361,7 @@ Changes * Set zoom to default instead of 100% with `:zoom`/`=`. * Adjust page zoom if default zoom changed. * Force tabs to be focused on `:undo`. -* Replace manual installation instructions on OS X with homebrew/macports. +* Replace manual installation instructions on macOS with homebrew/macports. * Allow min-/maximizing of print preview on Windows. * Various documentation improvements. * Various other small improvements and cleanups. diff --git a/CONTRIBUTING.asciidoc b/CONTRIBUTING.asciidoc index 2ac99804e..b7a13eaf8 100644 --- a/CONTRIBUTING.asciidoc +++ b/CONTRIBUTING.asciidoc @@ -700,7 +700,7 @@ as closed. * Linux: Run `python3 scripts/dev/build_release.py --upload v0.$x.$y` * Windows: Run `C:\Python36-32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v0.X.Y` (replace X/Y by hand) -* OS X: Run `python3 scripts/dev/build_release.py --upload v0.X.Y` (replace X/Y by hand) +* macOS: Run `python3 scripts/dev/build_release.py --upload v0.X.Y` (replace X/Y by hand) * On server: Run `python3 scripts/dev/download_release.py v0.X.Y` (replace X/Y by hand) * Update `qutebrowser-git` PKGBUILD if dependencies/install changed * Announce to qutebrowser and qutebrowser-announce mailinglist diff --git a/INSTALL.asciidoc b/INSTALL.asciidoc index 425bf738c..7f197d408 100644 --- a/INSTALL.asciidoc +++ b/INSTALL.asciidoc @@ -277,13 +277,13 @@ $ pip install tox Then <>. -On OS X -------- +On macOS +-------- Prebuilt binary ~~~~~~~~~~~~~~~ -The easiest way to install qutebrowser on OS X is to use the prebuilt `.app` +The easiest way to install qutebrowser on macOS is to use the prebuilt `.app` files from the https://github.com/qutebrowser/qutebrowser/releases[release page]. diff --git a/doc/quickstart.asciidoc b/doc/quickstart.asciidoc index 7d597ed2e..4881cca62 100644 --- a/doc/quickstart.asciidoc +++ b/doc/quickstart.asciidoc @@ -31,7 +31,7 @@ image:http://qutebrowser.org/img/cheatsheet-small.png["qutebrowser key binding c * Run `:adblock-update` to download adblock lists and activate adblocking. * If you just cloned the repository, you'll need to run `scripts/asciidoc2html.py` to generate the documentation. -* Go to the link:qute://settings[settings page] to set up qutebrowser the way you want it. (Currently not available with the QtWebEngine backend and on the OS X build - use the `:set` command instead) +* Go to the link:qute://settings[settings page] to set up qutebrowser the way you want it. (Currently not available with the QtWebEngine backend and on the macOS build - use the `:set` command instead) * Subscribe to https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[the mailinglist] or https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[the announce-only mailinglist]. diff --git a/doc/userscripts.asciidoc b/doc/userscripts.asciidoc index b44a6b8ff..85811266d 100644 --- a/doc/userscripts.asciidoc +++ b/doc/userscripts.asciidoc @@ -60,7 +60,7 @@ Sending commands Normal qutebrowser commands can be written to `$QUTE_FIFO` and will be executed. -On Unix/OS X, this is a named pipe and commands written to it will get executed +On Unix/macOS, this is a named pipe and commands written to it will get executed immediately. On Windows, this is a regular file, and the commands in it will be executed as diff --git a/pytest.ini b/pytest.ini index ab437801b..ad1a56086 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,8 +5,8 @@ markers = posix: Tests which only can run on a POSIX OS. windows: Tests which only can run on Windows. linux: Tests which only can run on Linux. - osx: Tests which only can run on OS X. - not_osx: Tests which can not run on OS X. + mac: Tests which only can run on macOS. + not_mac: Tests which can not run on macOS. not_frozen: Tests which can't be run if sys.frozen is True. no_xvfb: Tests which can't be run with Xvfb. frozen: Tests which can only be run if sys.frozen is True. @@ -20,7 +20,7 @@ markers = qtwebkit_ng_xfail: Tests failing with QtWebKit-NG qtwebkit_ng_skip: Tests skipped with QtWebKit-NG qtwebengine_flaky: Tests which are flaky (and currently skipped) with QtWebEngine - qtwebengine_osx_xfail: Tests which fail on OS X with QtWebEngine + qtwebengine_mac_xfail: Tests which fail on macOS with QtWebEngine js_prompt: Tests needing to display a javascript prompt this: Used to mark tests during development no_invalid_lines: Don't fail on unparseable lines in end2end tests diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index eeee223aa..86fefaeb3 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -499,7 +499,7 @@ class TabBar(QTabBar): width = int(confwidth) size = QSize(max(minimum_size.width(), width), height) elif self.count() == 0: - # This happens on startup on OS X. + # This happens on startup on macOS. # We return it directly rather than setting `size' because we don't # want to ensure it's valid in this special case. return QSize() diff --git a/qutebrowser/misc/ipc.py b/qutebrowser/misc/ipc.py index eb9aa4a3b..de22162be 100644 --- a/qutebrowser/misc/ipc.py +++ b/qutebrowser/misc/ipc.py @@ -434,12 +434,12 @@ def _has_legacy_server(name): log.ipc.debug("Socket error: {} ({})".format( socket.errorString(), err)) - os_x_fail = (sys.platform == 'darwin' and - socket.errorString() == 'QLocalSocket::connectToServer: ' - 'Unknown error 38') + mac_fail = (sys.platform == 'darwin' and + socket.errorString() == 'QLocalSocket::connectToServer: ' + 'Unknown error 38') if err not in [QLocalSocket.ServerNotFoundError, - QLocalSocket.ConnectionRefusedError] and not os_x_fail: + QLocalSocket.ConnectionRefusedError] and not mac_fail: return True socket.disconnectFromServer() diff --git a/qutebrowser/utils/standarddir.py b/qutebrowser/utils/standarddir.py index a1cedddd0..28d84764f 100644 --- a/qutebrowser/utils/standarddir.py +++ b/qutebrowser/utils/standarddir.py @@ -107,7 +107,7 @@ def runtime(): if sys.platform.startswith('linux'): typ = QStandardPaths.RuntimeLocation else: # pragma: no cover - # RuntimeLocation is a weird path on OS X and Windows. + # RuntimeLocation is a weird path on macOS and Windows. typ = QStandardPaths.TempLocation overridden, path = _from_args(typ, _args) diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py index 0b527637e..833744d9b 100644 --- a/qutebrowser/utils/utils.py +++ b/qutebrowser/utils/utils.py @@ -373,7 +373,7 @@ def keyevent_to_string(e): None if only modifiers are pressed.. """ if sys.platform == 'darwin': - # Qt swaps Ctrl/Meta on OS X, so we switch it back here so the user can + # Qt swaps Ctrl/Meta on macOS, so we switch it back here so the user can # use it in the config as expected. See: # https://github.com/qutebrowser/qutebrowser/issues/110 # http://doc.qt.io/qt-5.4/osx-issues.html#special-keys diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index bb5733cad..d5c03ad02 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -92,7 +92,7 @@ def smoke_test(executable): '--temp-basedir', 'about:blank', ':later 500 quit']) -def patch_osx_app(): +def patch_mac_app(): """Patch .app to copy missing data and link some libs. See https://github.com/pyinstaller/pyinstaller/issues/2276 @@ -125,8 +125,8 @@ def patch_osx_app(): os.path.join(dest, lib)) -def build_osx(): - """Build OS X .dmg/.app.""" +def build_mac(): + """Build macOS .dmg/.app.""" utils.print_title("Cleaning up...") for f in ['wc.dmg', 'template.dmg']: try: @@ -141,7 +141,7 @@ def build_osx(): utils.print_title("Building .app via pyinstaller") call_tox('pyinstaller', '-r') utils.print_title("Patching .app") - patch_osx_app() + patch_mac_app() utils.print_title("Building .dmg") subprocess.check_call(['make', '-f', 'scripts/dev/Makefile-dmg']) @@ -163,7 +163,7 @@ def build_osx(): except PermissionError as e: print("Failed to remove tempdir: {}".format(e)) - return [(dmg_name, 'application/x-apple-diskimage', 'OS X .dmg')] + return [(dmg_name, 'application/x-apple-diskimage', 'macOS .dmg')] def patch_windows(out_dir): @@ -366,7 +366,7 @@ def main(): artifacts = build_windows() elif sys.platform == 'darwin': run_asciidoc2html(args) - artifacts = build_osx() + artifacts = build_mac() else: artifacts = build_sdist() upload_to_pypi = True diff --git a/tests/conftest.py b/tests/conftest.py index 6ac9515ce..f2d6da8ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -50,8 +50,8 @@ def _apply_platform_markers(config, item): ('posix', os.name != 'posix', "Requires a POSIX os"), ('windows', os.name != 'nt', "Requires Windows"), ('linux', not sys.platform.startswith('linux'), "Requires Linux"), - ('osx', sys.platform != 'darwin', "Requires OS X"), - ('not_osx', sys.platform == 'darwin', "Skipped on OS X"), + ('mac', sys.platform != 'darwin', "Requires macOS"), + ('not_mac', sys.platform == 'darwin', "Skipped on macOS"), ('not_frozen', getattr(sys, 'frozen', False), "Can't be run when frozen"), ('frozen', not getattr(sys, 'frozen', False), diff --git a/tests/end2end/conftest.py b/tests/end2end/conftest.py index 8dac6a41d..75c6845f4 100644 --- a/tests/end2end/conftest.py +++ b/tests/end2end/conftest.py @@ -149,7 +149,7 @@ def pytest_collection_modifyitems(config, items): not config.webengine and qtutils.is_qtwebkit_ng()), ('qtwebengine_flaky', 'Flaky with QtWebEngine', pytest.mark.skipif, config.webengine), - ('qtwebengine_osx_xfail', 'Fails on OS X with QtWebEngine', + ('qtwebengine_mac_xfail', 'Fails on macOS with QtWebEngine', pytest.mark.xfail, config.webengine and sys.platform == 'darwin'), ] diff --git a/tests/end2end/features/conftest.py b/tests/end2end/features/conftest.py index 3b4652efc..b67f69418 100644 --- a/tests/end2end/features/conftest.py +++ b/tests/end2end/features/conftest.py @@ -61,7 +61,7 @@ def pytest_runtest_makereport(item, call): if (not hasattr(report.longrepr, 'addsection') or not hasattr(report, 'scenario')): - # In some conditions (on OS X and Windows it seems), report.longrepr is + # In some conditions (on macOS and Windows it seems), report.longrepr is # actually a tuple. This is handled similarily in pytest-qt too. # # Since this hook is invoked for any test, we also need to skip it for diff --git a/tests/end2end/features/hints.feature b/tests/end2end/features/hints.feature index a5e073d8a..4fbd17720 100644 --- a/tests/end2end/features/hints.feature +++ b/tests/end2end/features/hints.feature @@ -243,7 +243,7 @@ Feature: Using hints ### hints -> auto-follow-timeout - @not_osx + @not_mac Scenario: Ignoring key presses after auto-following hints When I set hints -> auto-follow-timeout to 1000 And I set hints -> mode to number diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index 279df7808..ca4fe5a3d 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -392,7 +392,7 @@ Feature: Various utility commands. And I run :debug-pyeval QApplication.instance().activeModalWidget().close() Then no crash should happen - # On Windows/OS X, we get a "QPrintDialog: Cannot be used on non-native + # On Windows/macOS, we get a "QPrintDialog: Cannot be used on non-native # printers" qWarning. # # Disabled because it causes weird segfaults and QPainter warnings in Qt... diff --git a/tests/end2end/features/prompts.feature b/tests/end2end/features/prompts.feature index 9b5d7aae5..e733d69b1 100644 --- a/tests/end2end/features/prompts.feature +++ b/tests/end2end/features/prompts.feature @@ -219,14 +219,14 @@ Feature: Prompts And I run :click-element id button Then the javascript message "geolocation permission denied" should be logged - @ci @not_osx @qt!=5.8 + @ci @not_mac @qt!=5.8 Scenario: Always accepting geolocation When I set content -> geolocation to true And I open data/prompt/geolocation.html in a new tab And I run :click-element id button Then the javascript message "geolocation permission denied" should not be logged - @ci @not_osx @qt!=5.8 + @ci @not_mac @qt!=5.8 Scenario: geolocation with ask -> true When I set content -> geolocation to ask And I open data/prompt/geolocation.html in a new tab diff --git a/tests/end2end/features/yankpaste.feature b/tests/end2end/features/yankpaste.feature index 9cf917c2c..ba5b56ca7 100644 --- a/tests/end2end/features/yankpaste.feature +++ b/tests/end2end/features/yankpaste.feature @@ -291,7 +291,7 @@ Feature: Yanking and pasting. # Compare Then the javascript message "textarea contents: onHello worlde two three four" should be logged - @qtwebengine_osx_xfail + @qtwebengine_mac_xfail Scenario: Inserting text into a text field with undo When I set general -> log-javascript-console to info And I open data/paste_primary.html diff --git a/tests/end2end/fixtures/testprocess.py b/tests/end2end/fixtures/testprocess.py index f94ec4e22..c0fc372cc 100644 --- a/tests/end2end/fixtures/testprocess.py +++ b/tests/end2end/fixtures/testprocess.py @@ -103,7 +103,7 @@ def pytest_runtest_makereport(item, call): httpbin_log = getattr(item, '_httpbin_log', None) if not hasattr(report.longrepr, 'addsection'): - # In some conditions (on OS X and Windows it seems), report.longrepr is + # In some conditions (on macOS and Windows it seems), report.longrepr is # actually a tuple. This is handled similarily in pytest-qt too. return diff --git a/tests/unit/misc/test_ipc.py b/tests/unit/misc/test_ipc.py index e364d6cc6..5249a9d55 100644 --- a/tests/unit/misc/test_ipc.py +++ b/tests/unit/misc/test_ipc.py @@ -207,9 +207,9 @@ class TestSocketName: socketname = ipc._get_socketname(basedir) assert socketname == expected - @pytest.mark.osx + @pytest.mark.mac @pytest.mark.parametrize('basedir, expected', POSIX_TESTS) - def test_os_x(self, basedir, expected): + def test_mac(self, basedir, expected): socketname = ipc._get_socketname(basedir) parts = socketname.split(os.sep) assert parts[-2] == 'qute_test' @@ -223,7 +223,7 @@ class TestSocketName: assert socketname == expected_path def test_other_unix(self): - """Fake test for POSIX systems which aren't Linux/OS X. + """Fake test for POSIX systems which aren't Linux/macOS. We probably would adjust the code first to make it work on that platform. @@ -512,7 +512,7 @@ class TestSendToRunningInstance: assert msg == "No existing instance present (error 2)" @pytest.mark.parametrize('has_cwd', [True, False]) - @pytest.mark.linux(reason="Causes random trouble on Windows and OS X") + @pytest.mark.linux(reason="Causes random trouble on Windows and macOS") def test_normal(self, qtbot, tmpdir, ipc_server, mocker, has_cwd): ipc_server.listen() @@ -562,7 +562,7 @@ class TestSendToRunningInstance: ipc.send_to_running_instance('qute-test', [], None, socket=socket) -@pytest.mark.not_osx(reason="https://github.com/qutebrowser/qutebrowser/" +@pytest.mark.not_mac(reason="https://github.com/qutebrowser/qutebrowser/" "issues/975") def test_timeout(qtbot, caplog, qlocalsocket, ipc_server): ipc_server._timer.setInterval(100) @@ -637,7 +637,7 @@ class TestSendOrListen: yield legacy_server legacy_server.shutdown() - @pytest.mark.linux(reason="Flaky on Windows and OS X") + @pytest.mark.linux(reason="Flaky on Windows and macOS") def test_normal_connection(self, caplog, qtbot, args): ret_server = ipc.send_or_listen(args) assert isinstance(ret_server, ipc.IPCServer) @@ -812,7 +812,7 @@ class TestSendOrListen: @pytest.mark.windows -@pytest.mark.osx +@pytest.mark.mac def test_long_username(monkeypatch): """See https://github.com/qutebrowser/qutebrowser/issues/888.""" username = 'alexandercogneau' diff --git a/tests/unit/utils/test_qtutils.py b/tests/unit/utils/test_qtutils.py index 99e1091b4..ff8b81c9a 100644 --- a/tests/unit/utils/test_qtutils.py +++ b/tests/unit/utils/test_qtutils.py @@ -550,7 +550,7 @@ if test_file is not None and sys.platform != 'darwin': # here which defines unittest TestCases to run the python tests over # PyQIODevice. - # Those are not run on OS X because that seems to cause a hang sometimes. + # Those are not run on macOS because that seems to cause a hang sometimes. @pytest.fixture(scope='session', autouse=True) def clean_up_python_testfile(): diff --git a/tests/unit/utils/test_standarddir.py b/tests/unit/utils/test_standarddir.py index c21ff7416..ea3368862 100644 --- a/tests/unit/utils/test_standarddir.py +++ b/tests/unit/utils/test_standarddir.py @@ -167,8 +167,8 @@ class TestStandardDir: (standarddir.cache, 2, ['Caches', 'qute_test']), (standarddir.download, 1, ['Downloads']), ]) - @pytest.mark.osx - def test_os_x(self, func, elems, expected): + @pytest.mark.mac + def test_mac(self, func, elems, expected): assert func().split(os.sep)[-elems:] == expected diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 6717cf75b..4c9baf6eb 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -668,8 +668,8 @@ class TestOsInfo: (('', ('', '', ''), ''), ''), (('x', ('1', '2', '3'), 'y'), 'x, 1.2.3, y'), ]) - def test_os_x_fake(self, monkeypatch, mac_ver, mac_ver_str): - """Test with a fake OS X. + def test_mac_fake(self, monkeypatch, mac_ver, mac_ver_str): + """Test with a fake macOS. Args: mac_ver: The tuple to set platform.mac_ver() to. @@ -698,9 +698,9 @@ class TestOsInfo: """Make sure there are no exceptions with a real Windows.""" version._os_info() - @pytest.mark.osx - def test_os_x_real(self): - """Make sure there are no exceptions with a real OS X.""" + @pytest.mark.mac + def test_mac_real(self): + """Make sure there are no exceptions with a real macOS.""" version._os_info() From f9f8900fe90c48e7a36d2d4108c25056275d23c7 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 7 Jul 2017 21:16:50 -0400 Subject: [PATCH 203/337] More sql code review fixes. - remove outdated comment - fix sql init error message - clean up history text import code - fix test_history file path in coverage check - use real web history, not stub, for completion model tests - use qtmodeltester in sql/list_category tests - test url encoding in history tests - fix test_clear by using a callable mock - remove test_debug_dump_history_oserror as the check is now the same as for the file not existing - rename nonempty to data in test_completionmodel - add more delete_cur_item tests - test empty option/value completion --- qutebrowser/app.py | 6 +- qutebrowser/browser/history.py | 11 +--- scripts/dev/check_coverage.py | 6 +- tests/helpers/fixtures.py | 28 ++++++--- tests/helpers/stubs.py | 32 ---------- tests/helpers/utils.py | 13 ---- tests/unit/browser/test_history.py | 31 +++++----- tests/unit/completion/test_completionmodel.py | 12 ++-- tests/unit/completion/test_listcategory.py | 26 +++++++- tests/unit/completion/test_models.py | 60 ++++++++++++------- tests/unit/completion/test_sqlcategory.py | 13 ++-- 11 files changed, 115 insertions(+), 123 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 46c7578bd..da66efe24 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -83,8 +83,6 @@ def run(args): standarddir.init(args) if args.version: - # we need to init sql to print the sql version - # we can use an in-memory database as we just want to query the version print(version.version()) sys.exit(usertypes.Exit.ok) @@ -431,8 +429,8 @@ def _init_modules(args, crash_handler): try: sql.init(os.path.join(standarddir.data(), 'history.sqlite')) except sql.SqlException as e: - error.handle_fatal_exc(e, args, 'Is sqlite installed?', - pre_text='Failed to initialize SQL') + error.handle_fatal_exc(e, args, 'Error initializing SQL', + pre_text='Error initializing SQL') sys.exit(usertypes.Exit.err_init) log.init.debug("Initializing web history...") diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index e583fba40..b5b8af149 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -171,6 +171,8 @@ class WebHistory(sql.SqlTable): def _parse_entry(self, line): """Parse a history line like '12345 http://example.com title'.""" + if not line or line.startswith('#'): + return None data = line.split(maxsplit=2) if len(data) == 2: atime, url = data @@ -240,11 +242,8 @@ class WebHistory(sql.SqlTable): data = {'url': [], 'title': [], 'atime': [], 'redirect': []} completion_data = {'url': [], 'title': [], 'last_atime': []} for (i, line) in enumerate(f): - line = line.strip() - if not line or line.startswith('#'): - continue try: - parsed = self._parse_entry(line) + parsed = self._parse_entry(line.strip()) if parsed is None: continue url, title, atime, redirect = parsed @@ -271,10 +270,6 @@ class WebHistory(sql.SqlTable): """ dest = os.path.expanduser(dest) - dirname = os.path.dirname(dest) - if not os.path.exists(dirname): - raise cmdexc.CommandError('Path does not exist', dirname) - lines = ('{}{} {} {}' .format(int(x.atime), '-r' * x.redirect, x.url, x.title) for x in self.select(sort_by='atime', sort_order='asc')) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index c01ae64af..950f21ff1 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -51,12 +51,10 @@ PERFECT_FILES = [ 'browser/webkit/cache.py'), ('tests/unit/browser/webkit/test_cookies.py', 'browser/webkit/cookies.py'), - ('tests/unit/browser/webkit/test_history.py', + ('tests/unit/browser/test_history.py', 'browser/history.py'), - ('tests/unit/browser/webkit/test_history.py', + ('tests/unit/browser/test_history.py', 'browser/webkit/webkithistory.py'), - ('tests/unit/browser/webkit/test_history.py', - 'browser/history.py'), ('tests/unit/browser/webkit/http/test_http.py', 'browser/webkit/http.py'), ('tests/unit/browser/webkit/http/test_content_disposition.py', diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 4fd6a0731..dd689a531 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -257,15 +257,6 @@ def bookmark_manager_stub(stubs): objreg.delete('bookmark-manager') -@pytest.fixture -def web_history_stub(init_sql, stubs): - """Fixture which provides a fake web-history object.""" - stub = stubs.WebHistoryStub() - objreg.register('web-history', stub) - yield stub - objreg.delete('web-history') - - @pytest.fixture def session_manager_stub(stubs): """Fixture which provides a fake session-manager object.""" @@ -491,3 +482,22 @@ def init_sql(data_tmpdir): sql.init(path) yield sql.close() + + +@pytest.fixture +def validate_model(qtmodeltester): + """Provides a function to validate a completion category.""" + def validate(cat, expected): + """Check that a category contains the items in the given order. + + Args: + cat: The category to inspect. + expected: A list of tuples containing the expected items. + """ + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(cat) + assert cat.rowCount() == len(expected) + for row, items in enumerate(expected): + for col, item in enumerate(items): + assert cat.data(cat.index(row, col)) == item + return validate diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 61dcefd66..47d254f86 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -33,7 +33,6 @@ from qutebrowser.browser import browsertab from qutebrowser.config import configexc from qutebrowser.utils import usertypes, utils from qutebrowser.mainwindow import mainwindow -from qutebrowser.misc import sql class FakeNetworkCache(QAbstractNetworkCache): @@ -523,37 +522,6 @@ class QuickmarkManagerStub(UrlMarkManagerStub): self.delete(key) -class WebHistoryStub(sql.SqlTable): - - """Stub for the web-history object.""" - - def __init__(self): - super().__init__("History", ['url', 'title', 'atime', 'redirect']) - self.completion = sql.SqlTable("CompletionHistory", - ['url', 'title', 'last_atime']) - - def __contains__(self, url): - q = self.contains_query('url') - return q.run(val=url).value() - - def add_url(self, url, title="", *, redirect=False, atime=None): - self.insert({'url': url, 'title': title, 'atime': atime, - 'redirect': redirect}) - if not redirect: - self.completion.insert({'url': url, - 'title': title, - 'last_atime': atime}) - - def delete_url(self, url): - """Remove all history entries with the given url. - - Args: - url: URL string to delete. - """ - self.delete('url', url) - self.completion.delete('url', url) - - class HostBlockerStub: """Stub for the host-blocker object.""" diff --git a/tests/helpers/utils.py b/tests/helpers/utils.py index 051b7bd50..7dbd7dd25 100644 --- a/tests/helpers/utils.py +++ b/tests/helpers/utils.py @@ -170,16 +170,3 @@ def abs_datapath(): """Get the absolute path to the end2end data directory.""" file_abs = os.path.abspath(os.path.dirname(__file__)) return os.path.join(file_abs, '..', 'end2end', 'data') - - -def validate_model(cat, expected): - """Check that a category contains the expected items in the given order. - - Args: - cat: The category to inspect. - expected: A list of tuples containing the expected items. - """ - assert cat.rowCount() == len(expected) - for row, items in enumerate(expected): - for col, item in enumerate(items): - assert cat.data(cat.index(row, col)) == item diff --git a/tests/unit/browser/test_history.py b/tests/unit/browser/test_history.py index 9496accd2..911184dc0 100644 --- a/tests/unit/browser/test_history.py +++ b/tests/unit/browser/test_history.py @@ -21,6 +21,7 @@ import logging import os +import unittest.mock import pytest from PyQt5.QtCore import QUrl @@ -114,7 +115,7 @@ def test_clear(qtbot, tmpdir, hist, mocker): hist.add_url(QUrl('http://www.qutebrowser.org/')) m = mocker.patch('qutebrowser.browser.history.message.confirm_async', - spec=[]) + new=unittest.mock.Mock, spec=[]) hist.clear() assert m.called @@ -134,16 +135,22 @@ def test_delete_url(hist): before = set(hist) hist.delete_url(QUrl('http://example.com/1')) diff = before.difference(set(hist)) - assert diff == set([('http://example.com/1', '', 0, False)]) + assert diff == {('http://example.com/1', '', 0, False)} -@pytest.mark.parametrize('url, atime, title, redirect', [ - ('http://www.example.com', 12346, 'the title', False), - ('http://www.example.com', 12346, 'the title', True) +@pytest.mark.parametrize('url, atime, title, redirect, expected_url', [ + ('http://www.example.com', 12346, 'the title', False, + 'http://www.example.com'), + ('http://www.example.com', 12346, 'the title', True, + 'http://www.example.com'), + ('http://www.example.com/spa ce', 12346, 'the title', False, + 'http://www.example.com/spa%20ce'), + ('https://user:pass@example.com', 12346, 'the title', False, + 'https://user@example.com'), ]) -def test_add_item(qtbot, hist, url, atime, title, redirect): +def test_add_item(qtbot, hist, url, atime, title, redirect, expected_url): hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) - assert list(hist) == [(url, title, atime, redirect)] + assert list(hist) == [(expected_url, title, atime, redirect)] def test_add_item_invalid(qtbot, hist, caplog): @@ -164,7 +171,7 @@ def test_add_item_invalid(qtbot, hist, caplog): def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog): with caplog.at_level(level): hist.add_from_tab(QUrl(url), QUrl(req_url), 'title') - assert set(list(hist)) == set(expected) + assert set(hist) == set(expected) @pytest.fixture @@ -330,11 +337,3 @@ def test_debug_dump_history_nonexistent(hist, tmpdir): histfile = tmpdir / 'nonexistent' / 'history' with pytest.raises(cmdexc.CommandError): hist.debug_dump_history(str(histfile)) - - -def test_debug_dump_history_oserror(hist, tmpdir): - histfile = tmpdir / 'history' - histfile.write('') - os.chmod(str(histfile), 0) - with pytest.raises(cmdexc.CommandError): - hist.debug_dump_history(str(histfile)) diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 2c0af3a4b..ce0fe3765 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -39,18 +39,18 @@ def test_first_last_item(counts): cat = mock.Mock(spec=['layoutChanged']) cat.rowCount = mock.Mock(return_value=c, spec=[]) model.add_category(cat) - nonempty = [i for i, rowCount in enumerate(counts) if rowCount > 0] - if not nonempty: + data = [i for i, rowCount in enumerate(counts) if rowCount > 0] + if not data: # with no items, first and last should be an invalid index assert not model.first_item().isValid() assert not model.last_item().isValid() else: - first = nonempty[0] - last = nonempty[-1] - # first item of the first nonempty category + first = data[0] + last = data[-1] + # first item of the first data category assert model.first_item().row() == 0 assert model.first_item().parent().row() == first - # last item of the last nonempty category + # last item of the last data category assert model.last_item().row() == counts[last] - 1 assert model.last_item().parent().row() == last diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index 5b6a5eea4..06c7ac1a6 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -19,10 +19,12 @@ """Tests for CompletionFilterModel.""" +from unittest import mock + import pytest -from helpers import utils from qutebrowser.completion.models import listcategory +from qutebrowser.commands import cmdexc @pytest.mark.parametrize('pattern, before, after', [ @@ -43,8 +45,26 @@ from qutebrowser.completion.models import listcategory [('foo', 'bar'), ('bar', 'foo'), ('bar', 'bar')], [('foo', 'bar'), ('bar', 'foo')]), ]) -def test_set_pattern(pattern, before, after): +def test_set_pattern(pattern, before, after, validate_model): """Validate the filtering and sorting results of set_pattern.""" cat = listcategory.ListCategory('Foo', before) cat.set_pattern(pattern) - utils.validate_model(cat, after) + validate_model(cat, after) + + +def test_delete_cur_item(validate_model): + func = mock.Mock(spec=[]) + cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')], + delete_func=func) + idx = cat.index(0, 0) + cat.delete_cur_item(idx) + func.assert_called_once_with(['a', 'b']) + validate_model(cat, [('c', 'd')]) + + +def test_delete_cur_item_no_func(validate_model): + cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')]) + idx = cat.index(0, 0) + with pytest.raises(cmdexc.CommandError, match="Cannot delete this item."): + cat.delete_cur_item(idx) + validate_model(cat, [('a', 'b'), ('c', 'd')]) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index b7b688a81..4fab0c402 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -27,7 +27,8 @@ from PyQt5.QtCore import QUrl from qutebrowser.completion.models import miscmodels, urlmodel, configmodel from qutebrowser.config import sections, value -from qutebrowser.misc import sql +from qutebrowser.utils import objreg +from qutebrowser.browser import history def _check_completions(model, expected): @@ -136,29 +137,33 @@ def bookmarks(bookmark_manager_stub): @pytest.fixture -def history_completion_table(init_sql): - return sql.SqlTable("CompletionHistory", ['url', 'title', 'last_atime']) +def web_history(init_sql, stubs): + """Fixture which provides a web-history object.""" + stub = history.WebHistory() + objreg.register('web-history', stub) + yield stub + objreg.delete('web-history') @pytest.fixture -def web_history(web_history_stub): +def web_history_populated(web_history): """Pre-populate the web-history database.""" - web_history_stub.add_url( - url='http://qutebrowser.org', + web_history.add_url( + url=QUrl('http://qutebrowser.org'), title='qutebrowser', atime=datetime(2015, 9, 5).timestamp() ) - web_history_stub.add_url( - url='https://python.org', + web_history.add_url( + url=QUrl('https://python.org'), title='Welcome to Python.org', atime=datetime(2016, 3, 8).timestamp() ) - web_history_stub.add_url( - url='https://github.com', + web_history.add_url( + url=QUrl('https://github.com'), title='https://github.com', atime=datetime(2016, 5, 1).timestamp() ) - return web_history_stub + return web_history def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub, @@ -261,8 +266,8 @@ def test_bookmark_completion(qtmodeltester, bookmarks): }) -def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, - bookmarks): +def test_url_completion(qtmodeltester, config_stub, web_history_populated, + quickmarks, bookmarks): """Test the results of url completion. Verify that: @@ -313,12 +318,12 @@ def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks, ('foo%bar', '', '%', 1), ('foobar', '', '%', 0), ]) -def test_url_completion_pattern(config_stub, web_history_stub, +def test_url_completion_pattern(config_stub, web_history, quickmark_manager_stub, bookmark_manager_stub, url, title, pattern, rowcount): """Test that url completion filters by url and title.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} - web_history_stub.add_url(url, title) + web_history.add_url(QUrl(url), title) model = urlmodel.url() model.set_pattern(pattern) # 2, 0 is History @@ -349,7 +354,7 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, bookmarks, def test_url_completion_delete_quickmark(qtmodeltester, config_stub, - web_history, quickmarks, bookmarks, + quickmarks, web_history, bookmarks, qtbot): """Test deleting a bookmark from the url completion model.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} @@ -373,7 +378,7 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, def test_url_completion_delete_history(qtmodeltester, config_stub, - web_history_stub, web_history, + web_history_populated, quickmarks, bookmarks): """Test deleting a history entry.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} @@ -389,9 +394,9 @@ def test_url_completion_delete_history(qtmodeltester, config_stub, assert model.data(parent) == "History" assert model.data(idx) == 'https://python.org' - assert 'https://python.org' in web_history_stub + assert 'https://python.org' in web_history_populated model.delete_cur_item(idx) - assert 'https://python.org' not in web_history_stub + assert 'https://python.org' not in web_history_populated def test_session_completion(qtmodeltester, session_manager_stub): @@ -504,6 +509,12 @@ def test_setting_option_completion(qtmodeltester, monkeypatch, stubs, }) +def test_setting_option_completion_empty(monkeypatch, stubs, config_stub): + module = 'qutebrowser.completion.models.configmodel' + _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') + assert configmodel.option('typo') is None + + def test_setting_option_completion_valuelist(qtmodeltester, monkeypatch, stubs, config_stub): module = 'qutebrowser.completion.models.configmodel' @@ -545,6 +556,13 @@ def test_setting_value_completion(qtmodeltester, monkeypatch, stubs, }) +def test_setting_value_completion_empty(monkeypatch, stubs, config_stub): + module = 'qutebrowser.completion.models.configmodel' + _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA') + config_stub.data = {'general': {}} + assert configmodel.value('general', 'typo') is None + + def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, key_config_stub): """Test the results of keybinding command completion. @@ -583,7 +601,7 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, def test_url_completion_benchmark(benchmark, config_stub, quickmark_manager_stub, bookmark_manager_stub, - web_history_stub): + web_history): """Benchmark url completion.""" config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', 'web-history-max-items': 1000} @@ -595,7 +613,7 @@ def test_url_completion_benchmark(benchmark, config_stub, 'title': ['title{}'.format(i) for i in r] } - web_history_stub.completion.insert_batch(entries) + web_history.completion.insert_batch(entries) quickmark_manager_stub.marks = collections.OrderedDict([ ('title{}'.format(i), 'example.com/{}'.format(i)) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 10d96f571..c20f8f1c5 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -23,7 +23,6 @@ import unittest.mock import pytest -from helpers import utils from qutebrowser.misc import sql from qutebrowser.completion.models import sqlcategory from qutebrowser.commands import cmdexc @@ -61,14 +60,14 @@ pytestmark = pytest.mark.usefixtures('init_sql') [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), ]) -def test_sorting(sort_by, sort_order, data, expected): +def test_sorting(sort_by, sort_order, data, expected, validate_model): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) cat.set_pattern('') - utils.validate_model(cat, expected) + validate_model(cat, expected) @pytest.mark.parametrize('pattern, filter_cols, before, after', [ @@ -116,7 +115,7 @@ def test_sorting(sort_by, sort_order, data, expected): [("can't touch this", '', ''), ('a', '', '')], [("can't touch this", '', '')]), ]) -def test_set_pattern(pattern, filter_cols, before, after): +def test_set_pattern(pattern, filter_cols, before, after, validate_model): """Validate the filtering and sorting results of set_pattern.""" table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: @@ -124,15 +123,15 @@ def test_set_pattern(pattern, filter_cols, before, after): filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) cat.set_pattern(pattern) - utils.validate_model(cat, after) + validate_model(cat, after) -def test_select(): +def test_select(validate_model): table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') cat.set_pattern('') - utils.validate_model(cat, [('bar', 'baz', 'foo')]) + validate_model(cat, [('bar', 'baz', 'foo')]) def test_delete_cur_item(): From c9fd182dba0e922550873a13c7db825dfee774b7 Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Sat, 8 Jul 2017 16:28:58 +0200 Subject: [PATCH 204/337] Adjust suggested_fn_from_title, add tests --- qutebrowser/browser/downloads.py | 4 ++-- tests/unit/browser/webkit/test_downloads.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py index e9499600e..438dfc528 100644 --- a/qutebrowser/browser/downloads.py +++ b/qutebrowser/browser/downloads.py @@ -197,8 +197,8 @@ def suggested_fn_from_title(url_path, title=None): _, ext = os.path.splitext(url_path) if ext.lower() in ext_whitelist and title: suggested_fn = utils.sanitize_filename(title) - if not suggested_fn.lower().endswith(ext.lower()): - suggested_fn += ext + if not suggested_fn.lower().endswith((".html", ".htm")): + suggested_fn += ".html" else: suggested_fn = None return suggested_fn diff --git a/tests/unit/browser/webkit/test_downloads.py b/tests/unit/browser/webkit/test_downloads.py index c9451b949..5a214f638 100644 --- a/tests/unit/browser/webkit/test_downloads.py +++ b/tests/unit/browser/webkit/test_downloads.py @@ -39,16 +39,22 @@ def test_download_model(qapp, qtmodeltester, config_stub, cookiejar_and_cache): 'Installing qutebrowser _ qutebrowser.html'), ('http://qutebrowser.org/INSTALL.HTML', 'Installing qutebrowser | qutebrowser', - 'Installing qutebrowser _ qutebrowser.HTML'), + 'Installing qutebrowser _ qutebrowser.html'), ('http://qutebrowser.org/INSTALL.html', 'Installing qutebrowser | qutebrowser.HTML', 'Installing qutebrowser _ qutebrowser.HTML'), ('http://qutebrowser.org/', 'qutebrowser | qutebrowser', - 'qutebrowser _ qutebrowser'), + 'qutebrowser _ qutebrowser.html'), ('https://github.com/qutebrowser/qutebrowser/releases', 'Releases · qutebrowser/qutebrowser', - 'Releases · qutebrowser_qutebrowser'), + 'Releases · qutebrowser_qutebrowser.html'), + ('http://qutebrowser.org/index.php', + 'qutebrowser | qutebrowser', + 'qutebrowser _ qutebrowser.html'), + ('http://qutebrowser.org/index.php', + 'qutebrowser | qutebrowser - index.php', + 'qutebrowser _ qutebrowser - index.php.html'), ('https://qutebrowser.org/img/cheatsheet-big.png', 'cheatsheet-big.png (3342×2060)', None), From b81474d2fd3851c4e08bb5e2a45f8ca9ca27e44f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sat, 8 Jul 2017 17:33:50 +0200 Subject: [PATCH 205/337] Improve earlyinit check for PyOpenGL Importing OpenGL alone doesn't actually load libgl, it only checks that the package is here. If libgl is missing, we'd later get an exception. --- qutebrowser/misc/earlyinit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index 4993d6927..f5145e015 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -342,7 +342,7 @@ def check_libraries(backend): modules['PyQt5.QtWebEngineWidgets'] = _missing_str("QtWebEngine", webengine=True) modules['PyQt5.QtOpenGL'] = _missing_str("PyQt5.QtOpenGL") - modules['OpenGL'] = _missing_str("PyOpenGL") + modules['OpenGL.GL'] = _missing_str("PyOpenGL") else: assert backend == 'webkit' modules['PyQt5.QtWebKit'] = _missing_str("PyQt5.QtWebKit") From fcf5158258be10194d59486538d4bf091a6a035a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sat, 8 Jul 2017 17:36:14 +0200 Subject: [PATCH 206/337] Recommend QT_XCB_FORCE_SOFTWARE_OPENGL This won't disable OpenGL for stuff started from qutebrowser. See #2368. --- qutebrowser/browser/webengine/webenginetab.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index ab4500364..11672d3b3 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -51,12 +51,13 @@ def init(): global _qute_scheme_handler app = QApplication.instance() - software_rendering = os.environ.get('LIBGL_ALWAYS_SOFTWARE') == '1' + software_rendering = (os.environ.get('LIBGL_ALWAYS_SOFTWARE') == '1' or + 'QT_XCB_FORCE_SOFTWARE_OPENGL' in os.environ) if version.opengl_vendor() == 'nouveau' and not software_rendering: # FIXME:qtwebengine display something more sophisticated here raise browsertab.WebTabError( "QtWebEngine is not supported with Nouveau graphics (unless " - "LIBGL_ALWAYS_SOFTWARE is set as environment variable).") + "QT_XCB_FORCE_SOFTWARE_OPENGL is set as environment variable).") log.init.debug("Initializing qute://* handler...") _qute_scheme_handler = webenginequtescheme.QuteSchemeHandler(parent=app) From cf4ac1a5b71c23a7b1e3aecb7fc110f5cde81f86 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 8 Jul 2017 16:34:38 -0400 Subject: [PATCH 207/337] SQL code review changes. - use mocker.Mock instead of mock.Mock to avoid an extra import - attach model to validator sooner so it can validate changes in the model during the test --- tests/helpers/fixtures.py | 33 +++++++++++++--------- tests/unit/browser/test_history.py | 3 +- tests/unit/completion/test_listcategory.py | 15 ++++++---- tests/unit/completion/test_sqlcategory.py | 15 ++++++---- 4 files changed, 38 insertions(+), 28 deletions(-) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index dd689a531..e38a3ce49 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -484,20 +484,25 @@ def init_sql(data_tmpdir): sql.close() -@pytest.fixture -def validate_model(qtmodeltester): - """Provides a function to validate a completion category.""" - def validate(cat, expected): - """Check that a category contains the items in the given order. +class ModelValidator: - Args: - cat: The category to inspect. - expected: A list of tuples containing the expected items. - """ - qtmodeltester.data_display_may_return_none = True - qtmodeltester.check(cat) - assert cat.rowCount() == len(expected) + """Validates completion models.""" + + def __init__(self, modeltester): + modeltester.data_display_may_return_none = True + self._modeltester = modeltester + + def set_model(self, model): + self._model = model + self._modeltester.check(model) + + def validate(self, expected): + assert self._model.rowCount() == len(expected) for row, items in enumerate(expected): for col, item in enumerate(items): - assert cat.data(cat.index(row, col)) == item - return validate + assert self._model.data(self._model.index(row, col)) == item + + +@pytest.fixture +def model_validator(qtmodeltester): + return ModelValidator(qtmodeltester) diff --git a/tests/unit/browser/test_history.py b/tests/unit/browser/test_history.py index 911184dc0..0b5e6eb41 100644 --- a/tests/unit/browser/test_history.py +++ b/tests/unit/browser/test_history.py @@ -21,7 +21,6 @@ import logging import os -import unittest.mock import pytest from PyQt5.QtCore import QUrl @@ -115,7 +114,7 @@ def test_clear(qtbot, tmpdir, hist, mocker): hist.add_url(QUrl('http://www.qutebrowser.org/')) m = mocker.patch('qutebrowser.browser.history.message.confirm_async', - new=unittest.mock.Mock, spec=[]) + new=mocker.Mock, spec=[]) hist.clear() assert m.called diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index 06c7ac1a6..3b1c1478a 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -45,26 +45,29 @@ from qutebrowser.commands import cmdexc [('foo', 'bar'), ('bar', 'foo'), ('bar', 'bar')], [('foo', 'bar'), ('bar', 'foo')]), ]) -def test_set_pattern(pattern, before, after, validate_model): +def test_set_pattern(pattern, before, after, model_validator): """Validate the filtering and sorting results of set_pattern.""" cat = listcategory.ListCategory('Foo', before) + model_validator.set_model(cat) cat.set_pattern(pattern) - validate_model(cat, after) + model_validator.validate(after) -def test_delete_cur_item(validate_model): +def test_delete_cur_item(model_validator): func = mock.Mock(spec=[]) cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')], delete_func=func) + model_validator.set_model(cat) idx = cat.index(0, 0) cat.delete_cur_item(idx) func.assert_called_once_with(['a', 'b']) - validate_model(cat, [('c', 'd')]) + model_validator.validate([('c', 'd')]) -def test_delete_cur_item_no_func(validate_model): +def test_delete_cur_item_no_func(model_validator): cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')]) + model_validator.set_model(cat) idx = cat.index(0, 0) with pytest.raises(cmdexc.CommandError, match="Cannot delete this item."): cat.delete_cur_item(idx) - validate_model(cat, [('a', 'b'), ('c', 'd')]) + model_validator.validate([('a', 'b'), ('c', 'd')]) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index c20f8f1c5..851bad257 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -60,14 +60,15 @@ pytestmark = pytest.mark.usefixtures('init_sql') [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), ]) -def test_sorting(sort_by, sort_order, data, expected, validate_model): +def test_sorting(sort_by, sort_order, data, expected, model_validator): table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in data: table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, sort_order=sort_order) + model_validator.set_model(cat) cat.set_pattern('') - validate_model(cat, expected) + model_validator.validate(expected) @pytest.mark.parametrize('pattern, filter_cols, before, after', [ @@ -115,23 +116,25 @@ def test_sorting(sort_by, sort_order, data, expected, validate_model): [("can't touch this", '', ''), ('a', '', '')], [("can't touch this", '', '')]), ]) -def test_set_pattern(pattern, filter_cols, before, after, validate_model): +def test_set_pattern(pattern, filter_cols, before, after, model_validator): """Validate the filtering and sorting results of set_pattern.""" table = sql.SqlTable('Foo', ['a', 'b', 'c']) for row in before: table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) + model_validator.set_model(cat) cat.set_pattern(pattern) - validate_model(cat, after) + model_validator.validate(after) -def test_select(validate_model): +def test_select(model_validator): table = sql.SqlTable('Foo', ['a', 'b', 'c']) table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') + model_validator.set_model(cat) cat.set_pattern('') - validate_model(cat, [('bar', 'baz', 'foo')]) + model_validator.validate([('bar', 'baz', 'foo')]) def test_delete_cur_item(): From 915cd5f016f22fc8d8b60c435d7ab41898334ec0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 11:51:22 +0200 Subject: [PATCH 208/337] Fix long lines --- qutebrowser/utils/utils.py | 4 ++-- tests/end2end/features/conftest.py | 4 ++-- tests/end2end/fixtures/testprocess.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py index 833744d9b..c85c53f25 100644 --- a/qutebrowser/utils/utils.py +++ b/qutebrowser/utils/utils.py @@ -373,8 +373,8 @@ def keyevent_to_string(e): None if only modifiers are pressed.. """ if sys.platform == 'darwin': - # Qt swaps Ctrl/Meta on macOS, so we switch it back here so the user can - # use it in the config as expected. See: + # Qt swaps Ctrl/Meta on macOS, so we switch it back here so the user + # can use it in the config as expected. See: # https://github.com/qutebrowser/qutebrowser/issues/110 # http://doc.qt.io/qt-5.4/osx-issues.html#special-keys modmask2str = collections.OrderedDict([ diff --git a/tests/end2end/features/conftest.py b/tests/end2end/features/conftest.py index b67f69418..0a67742f9 100644 --- a/tests/end2end/features/conftest.py +++ b/tests/end2end/features/conftest.py @@ -61,8 +61,8 @@ def pytest_runtest_makereport(item, call): if (not hasattr(report.longrepr, 'addsection') or not hasattr(report, 'scenario')): - # In some conditions (on macOS and Windows it seems), report.longrepr is - # actually a tuple. This is handled similarily in pytest-qt too. + # In some conditions (on macOS and Windows it seems), report.longrepr + # is actually a tuple. This is handled similarily in pytest-qt too. # # Since this hook is invoked for any test, we also need to skip it for # non-BDD ones. diff --git a/tests/end2end/fixtures/testprocess.py b/tests/end2end/fixtures/testprocess.py index c0fc372cc..1a2c51baf 100644 --- a/tests/end2end/fixtures/testprocess.py +++ b/tests/end2end/fixtures/testprocess.py @@ -103,8 +103,8 @@ def pytest_runtest_makereport(item, call): httpbin_log = getattr(item, '_httpbin_log', None) if not hasattr(report.longrepr, 'addsection'): - # In some conditions (on macOS and Windows it seems), report.longrepr is - # actually a tuple. This is handled similarily in pytest-qt too. + # In some conditions (on macOS and Windows it seems), report.longrepr + # is actually a tuple. This is handled similarily in pytest-qt too. return if pytest.config.getoption('--capture') == 'no': From 9e7f2e470f3134eae93854a711a843e162b9b65c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 11:54:00 +0200 Subject: [PATCH 209/337] Move OpenGL workaround import OpenGL.GL gets imported in earlyinit already anyways, so we can move everything there. --- qutebrowser/browser/webengine/webenginesettings.py | 7 ------- qutebrowser/misc/earlyinit.py | 8 +++++++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 493a2a687..936c42033 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -200,13 +200,6 @@ def init(args): if args.enable_webengine_inspector: os.environ['QTWEBENGINE_REMOTE_DEBUGGING'] = str(utils.random_port()) - # Workaround for a black screen with some setups - # https://github.com/spyder-ide/spyder/issues/3226 - if not os.environ.get('QUTE_NO_OPENGL_WORKAROUND'): - # Hide "No OpenGL_accelerate module loaded: ..." message - logging.getLogger('OpenGL.acceleratesupport').propagate = False - from OpenGL import GL # pylint: disable=unused-variable - _init_profiles() # We need to do this here as a WORKAROUND for diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index f5145e015..59cf6a6bf 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -36,6 +36,7 @@ import traceback import signal import importlib import datetime +import logging try: import tkinter except ImportError: @@ -342,7 +343,12 @@ def check_libraries(backend): modules['PyQt5.QtWebEngineWidgets'] = _missing_str("QtWebEngine", webengine=True) modules['PyQt5.QtOpenGL'] = _missing_str("PyQt5.QtOpenGL") - modules['OpenGL.GL'] = _missing_str("PyOpenGL") + # Workaround for a black screen with some setups + # https://github.com/spyder-ide/spyder/issues/3226 + if not os.environ.get('QUTE_NO_OPENGL_WORKAROUND'): + # Hide "No OpenGL_accelerate module loaded: ..." message + logging.getLogger('OpenGL.acceleratesupport').propagate = False + modules['OpenGL.GL'] = _missing_str("PyOpenGL") else: assert backend == 'webkit' modules['PyQt5.QtWebKit'] = _missing_str("PyQt5.QtWebKit") From cfb169b5f00d52b20202aa9c13f21bce153fa34a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 12:40:16 +0200 Subject: [PATCH 210/337] Remove unused import --- qutebrowser/browser/webengine/webenginesettings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 936c42033..27ef60cb6 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -28,7 +28,6 @@ Module attributes: """ import os -import logging from PyQt5.QtGui import QFont from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile, From 6a2163d36f0b6c16d5fbfaff3be25ab2cdd3cdda Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 12:45:26 +0200 Subject: [PATCH 211/337] ipc: Remove support for connecting to legacy servers --- qutebrowser/misc/ipc.py | 64 +++++-------------------------- tests/unit/misc/test_ipc.py | 75 ++----------------------------------- 2 files changed, 14 insertions(+), 125 deletions(-) diff --git a/qutebrowser/misc/ipc.py b/qutebrowser/misc/ipc.py index de22162be..03ccab729 100644 --- a/qutebrowser/misc/ipc.py +++ b/qutebrowser/misc/ipc.py @@ -20,7 +20,6 @@ """Utilities for IPC with existing instances.""" import os -import sys import time import json import getpass @@ -41,8 +40,8 @@ ATIME_INTERVAL = 60 * 60 * 3 * 1000 # 3 hours PROTOCOL_VERSION = 1 -def _get_socketname_legacy(basedir): - """Legacy implementation of _get_socketname.""" +def _get_socketname_windows(basedir): + """Get a socketname to use for Windows.""" parts = ['qutebrowser', getpass.getuser()] if basedir is not None: md5 = hashlib.md5(basedir.encode('utf-8')).hexdigest() @@ -50,10 +49,10 @@ def _get_socketname_legacy(basedir): return '-'.join(parts) -def _get_socketname(basedir, legacy=False): +def _get_socketname(basedir): """Get a socketname to use.""" - if legacy or os.name == 'nt': - return _get_socketname_legacy(basedir) + if os.name == 'nt': + return _get_socketname_windows(basedir) parts_to_hash = [getpass.getuser()] if basedir is not None: @@ -415,41 +414,7 @@ class IPCServer(QObject): self._remove_server() -def _has_legacy_server(name): - """Check if there is a legacy server. - - Args: - name: The name to try to connect to. - - Return: - True if there is a server with the given name, False otherwise. - """ - socket = QLocalSocket() - log.ipc.debug("Trying to connect to {}".format(name)) - socket.connectToServer(name) - - err = socket.error() - - if err != QLocalSocket.UnknownSocketError: - log.ipc.debug("Socket error: {} ({})".format( - socket.errorString(), err)) - - mac_fail = (sys.platform == 'darwin' and - socket.errorString() == 'QLocalSocket::connectToServer: ' - 'Unknown error 38') - - if err not in [QLocalSocket.ServerNotFoundError, - QLocalSocket.ConnectionRefusedError] and not mac_fail: - return True - - socket.disconnectFromServer() - if socket.state() != QLocalSocket.UnconnectedState: - socket.waitForDisconnected(CONNECT_TIMEOUT) - return False - - -def send_to_running_instance(socketname, command, target_arg, *, - legacy_name=None, socket=None): +def send_to_running_instance(socketname, command, target_arg, *, socket=None): """Try to send a commandline to a running instance. Blocks for CONNECT_TIMEOUT ms. @@ -459,7 +424,6 @@ def send_to_running_instance(socketname, command, target_arg, *, command: The command to send to the running instance. target_arg: --target command line argument socket: The socket to read data from, or None. - legacy_name: The legacy name to first try to connect to. Return: True if connecting was successful, False if no connection was made. @@ -467,13 +431,8 @@ def send_to_running_instance(socketname, command, target_arg, *, if socket is None: socket = QLocalSocket() - if legacy_name is not None and _has_legacy_server(legacy_name): - name_to_use = legacy_name - else: - name_to_use = socketname - - log.ipc.debug("Connecting to {}".format(name_to_use)) - socket.connectToServer(name_to_use) + log.ipc.debug("Connecting to {}".format(socketname)) + socket.connectToServer(socketname) connected = socket.waitForConnected(CONNECT_TIMEOUT) if connected: @@ -527,12 +486,10 @@ def send_or_listen(args): None if an instance was running and received our request. """ socketname = _get_socketname(args.basedir) - legacy_socketname = _get_socketname(args.basedir, legacy=True) try: try: sent = send_to_running_instance(socketname, args.command, - args.target, - legacy_name=legacy_socketname) + args.target) if sent: return None log.init.debug("Starting IPC server...") @@ -545,8 +502,7 @@ def send_or_listen(args): log.init.debug("Got AddressInUseError, trying again.") time.sleep(0.5) sent = send_to_running_instance(socketname, args.command, - args.target, - legacy_name=legacy_socketname) + args.target) if sent: return None else: diff --git a/tests/unit/misc/test_ipc.py b/tests/unit/misc/test_ipc.py index 5249a9d55..d090f8be4 100644 --- a/tests/unit/misc/test_ipc.py +++ b/tests/unit/misc/test_ipc.py @@ -182,11 +182,6 @@ def md5(inp): class TestSocketName: - LEGACY_TESTS = [ - (None, 'qutebrowser-testusername'), - ('/x', 'qutebrowser-testusername-{}'.format(md5('/x'))), - ] - POSIX_TESTS = [ (None, 'ipc-{}'.format(md5('testusername'))), ('/x', 'ipc-{}'.format(md5('testusername-/x'))), @@ -196,12 +191,10 @@ class TestSocketName: def patch_user(self, monkeypatch): monkeypatch.setattr(ipc.getpass, 'getuser', lambda: 'testusername') - @pytest.mark.parametrize('basedir, expected', LEGACY_TESTS) - def test_legacy(self, basedir, expected): - socketname = ipc._get_socketname(basedir, legacy=True) - assert socketname == expected - - @pytest.mark.parametrize('basedir, expected', LEGACY_TESTS) + @pytest.mark.parametrize('basedir, expected', [ + (None, 'qutebrowser-testusername'), + ('/x', 'qutebrowser-testusername-{}'.format(md5('/x'))), + ]) @pytest.mark.windows def test_windows(self, basedir, expected): socketname = ipc._get_socketname(basedir) @@ -629,14 +622,6 @@ class TestSendOrListen: setattr(m, attr, getattr(QLocalSocket, attr)) return m - @pytest.fixture - def legacy_server(self, args): - legacy_name = ipc._get_socketname(args.basedir, legacy=True) - legacy_server = ipc.IPCServer(legacy_name) - legacy_server.listen() - yield legacy_server - legacy_server.shutdown() - @pytest.mark.linux(reason="Flaky on Windows and macOS") def test_normal_connection(self, caplog, qtbot, args): ret_server = ipc.send_or_listen(args) @@ -651,54 +636,6 @@ class TestSendOrListen: assert ret_client is None - @pytest.mark.posix(reason="Unneeded on Windows") - def test_legacy_name(self, caplog, qtbot, args, legacy_server): - with qtbot.waitSignal(legacy_server.got_args): - ret = ipc.send_or_listen(args) - assert ret is None - msgs = [e.message for e in caplog.records] - assert "Connecting to {}".format(legacy_server._socketname) in msgs - - @pytest.mark.posix(reason="Unneeded on Windows") - def test_stale_legacy_server(self, caplog, qtbot, args, legacy_server, - ipc_server, py_proc): - legacy_name = ipc._get_socketname(args.basedir, legacy=True) - logging.debug('== Setting up the legacy server ==') - cmdline = py_proc(""" - import sys - - from PyQt5.QtCore import QCoreApplication - from PyQt5.QtNetwork import QLocalServer - - app = QCoreApplication([]) - - QLocalServer.removeServer(sys.argv[1]) - server = QLocalServer() - - ok = server.listen(sys.argv[1]) - assert ok - - print(server.fullServerName()) - """) - - name = subprocess.check_output( - [cmdline[0]] + cmdline[1] + [legacy_name]) - name = name.decode('utf-8').rstrip('\n') - - # Closing the server should not remove the FIFO yet - assert os.path.exists(name) - - ## Setting up the new server - logging.debug('== Setting up new server ==') - ret_server = ipc.send_or_listen(args) - assert isinstance(ret_server, ipc.IPCServer) - - logging.debug('== Connecting ==') - with qtbot.waitSignal(ret_server.got_args): - ret_client = ipc.send_or_listen(args) - - assert ret_client is None - @pytest.mark.posix(reason="Unneeded on Windows") def test_correct_socket_name(self, args): server = ipc.send_or_listen(args) @@ -723,9 +660,7 @@ class TestSendOrListen: qlocalsocket_mock().waitForConnected.side_effect = [False, True] qlocalsocket_mock().error.side_effect = [ - QLocalSocket.ServerNotFoundError, # legacy name QLocalSocket.ServerNotFoundError, - QLocalSocket.ServerNotFoundError, # legacy name QLocalSocket.UnknownSocketError, QLocalSocket.UnknownSocketError, # error() gets called twice ] @@ -761,10 +696,8 @@ class TestSendOrListen: # If it fails, that's the "not sent" case above. qlocalsocket_mock().waitForConnected.side_effect = [False, has_error] qlocalsocket_mock().error.side_effect = [ - QLocalSocket.ServerNotFoundError, # legacy name QLocalSocket.ServerNotFoundError, QLocalSocket.ServerNotFoundError, - QLocalSocket.ServerNotFoundError, # legacy name QLocalSocket.ConnectionRefusedError, QLocalSocket.ConnectionRefusedError, # error() gets called twice ] From cd063c74d9e050e8e44f8a8161c6d832a915ff3a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 12:54:19 +0200 Subject: [PATCH 212/337] Why is my commit -a broken --- CHANGELOG.asciidoc | 6 ++++++ tests/unit/misc/test_ipc.py | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 897afabf7..4fdb41fd9 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -36,6 +36,12 @@ Major changes the entire browsing history. - Completely rewritten configuration system. +Changed +~~~~~~~ + +- Upgrading qutebrowser with a version older than v0.4.0 still running now won't + work properly anymore. + Fixes ~~~~~ diff --git a/tests/unit/misc/test_ipc.py b/tests/unit/misc/test_ipc.py index d090f8be4..4a530a10b 100644 --- a/tests/unit/misc/test_ipc.py +++ b/tests/unit/misc/test_ipc.py @@ -26,7 +26,6 @@ import collections import logging import json import hashlib -import subprocess from unittest import mock import pytest From 6c0ceeac7f3a81f83c9c302bef913e05141ba2f0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 12:58:05 +0200 Subject: [PATCH 213/337] Update docs --- CHANGELOG.asciidoc | 1 + README.asciidoc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 4fdb41fd9..6b974e6ef 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -41,6 +41,7 @@ Changed - Upgrading qutebrowser with a version older than v0.4.0 still running now won't work properly anymore. +- Using `:download` now uses the page's title as filename. Fixes ~~~~~ diff --git a/README.asciidoc b/README.asciidoc index 3ed3bc054..972d03d51 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -181,6 +181,7 @@ Contributors, sorted by the number of commits in descending order: * Maciej Wołczyk * Clayton Craft * sandrosc +* Iordanis Grigoriou * Alexey "Averrin" Nabrodov * pkill9 * nanjekyejoannah @@ -239,7 +240,6 @@ Contributors, sorted by the number of commits in descending order: * Johannes Altmanninger * Jeremy Kaplan * Ismail -* Iordanis Grigoriou * Edgar Hipp * Daryl Finlay * arza From 38c00e53cdca519f35a550852af5ef3346e703bc Mon Sep 17 00:00:00 2001 From: Fritz Reichwald Date: Sun, 9 Jul 2017 13:34:10 +0200 Subject: [PATCH 214/337] Add open_url_in_instance.sh script --- scripts/open_url_in_instance.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 scripts/open_url_in_instance.sh diff --git a/scripts/open_url_in_instance.sh b/scripts/open_url_in_instance.sh new file mode 100755 index 000000000..119c3aa4f --- /dev/null +++ b/scripts/open_url_in_instance.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# initial idea: Florian Bruhin (The-Compiler) +# author: Thore Bödecker (foxxx0) + +_url="$1" +_qb_version='0.10.1' +_proto_version=1 +_ipc_socket="${XDG_RUNTIME_DIR}/qutebrowser/ipc-$(echo -n "$USER" | md5sum | cut -d' ' -f1)" + +if [[ -e "${_ipc_socket}" ]]; then + exec printf '{"args": ["%s"], "target_arg": null, "version": "%s", "protocol_version": %d, "cwd": "%s"}\n' \ + "${_url}" \ + "${_qb_version}" \ + "${_proto_version}" \ + "${PWD}" | socat - UNIX-CONNECT:"${_ipc_socket}" +else + exec /usr/bin/qutebrowser --backend webengine "$@" +fi From ead71db41a4c0f2484f39b98d55ab03951fac54e Mon Sep 17 00:00:00 2001 From: Fritz Reichwald Date: Sun, 9 Jul 2017 13:45:16 +0200 Subject: [PATCH 215/337] Add explanation for using open_url_in_instance script --- FAQ.asciidoc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/FAQ.asciidoc b/FAQ.asciidoc index 23061db44..d3eea17e3 100644 --- a/FAQ.asciidoc +++ b/FAQ.asciidoc @@ -171,6 +171,19 @@ What's the difference between insert and passthrough mode?:: be useful to rebind escape to something else in passthrough mode only, to be able to send an escape keypress to the website. +Why takes it longer to open an URL in qutebrowser than in chromium?:: + When opening an URL in an existing instance the normal qutebrowser + python script is started and a few PyQT libraries need to be + loaded until it is detected that there is an instance running + where the URL is then passed to. One workaround is to use this + https://github.com/qutebrowser/qutebrowser/blob/master/scripts/open_url_in_instance.sh[script] + and place it in your $PATH with the name "qutebrowser". This + script passes the URL via an unix socket to qutebrowser (if its + running already) using socat which is much faster and starts a new + qutebrowser if it is not running already. Also check if you want + to use webengine as backend in line 17 and change it to your + needs. + == Troubleshooting Configuration not saved after modifying config.:: From fd4bc29beb1bbfdd2d5586251bf48ca7bd3d79a6 Mon Sep 17 00:00:00 2001 From: Fritz Reichwald Date: Sun, 9 Jul 2017 14:10:08 +0200 Subject: [PATCH 216/337] Add some comment --- FAQ.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FAQ.asciidoc b/FAQ.asciidoc index d3eea17e3..8eba424de 100644 --- a/FAQ.asciidoc +++ b/FAQ.asciidoc @@ -175,7 +175,8 @@ Why takes it longer to open an URL in qutebrowser than in chromium?:: When opening an URL in an existing instance the normal qutebrowser python script is started and a few PyQT libraries need to be loaded until it is detected that there is an instance running - where the URL is then passed to. One workaround is to use this + where the URL is then passed to. This takes some time. + One workaround is to use this https://github.com/qutebrowser/qutebrowser/blob/master/scripts/open_url_in_instance.sh[script] and place it in your $PATH with the name "qutebrowser". This script passes the URL via an unix socket to qutebrowser (if its From bb567a61b62342aaa28e0dafe777768954dd4149 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 22:09:31 +0200 Subject: [PATCH 217/337] Fix ipc test coverage --- qutebrowser/misc/ipc.py | 2 +- tests/unit/misc/test_ipc.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/qutebrowser/misc/ipc.py b/qutebrowser/misc/ipc.py index 03ccab729..562cc84cc 100644 --- a/qutebrowser/misc/ipc.py +++ b/qutebrowser/misc/ipc.py @@ -51,7 +51,7 @@ def _get_socketname_windows(basedir): def _get_socketname(basedir): """Get a socketname to use.""" - if os.name == 'nt': + if os.name == 'nt': # pragma: no cover return _get_socketname_windows(basedir) parts_to_hash = [getpass.getuser()] diff --git a/tests/unit/misc/test_ipc.py b/tests/unit/misc/test_ipc.py index 4a530a10b..d0758b28d 100644 --- a/tests/unit/misc/test_ipc.py +++ b/tests/unit/misc/test_ipc.py @@ -186,19 +186,26 @@ class TestSocketName: ('/x', 'ipc-{}'.format(md5('testusername-/x'))), ] + WINDOWS_TESTS = [ + (None, 'qutebrowser-testusername'), + ('/x', 'qutebrowser-testusername-{}'.format(md5('/x'))), + ] + @pytest.fixture(autouse=True) def patch_user(self, monkeypatch): monkeypatch.setattr(ipc.getpass, 'getuser', lambda: 'testusername') - @pytest.mark.parametrize('basedir, expected', [ - (None, 'qutebrowser-testusername'), - ('/x', 'qutebrowser-testusername-{}'.format(md5('/x'))), - ]) + @pytest.mark.parametrize('basedir, expected', WINDOWS_TESTS) @pytest.mark.windows def test_windows(self, basedir, expected): socketname = ipc._get_socketname(basedir) assert socketname == expected + @pytest.mark.parametrize('basedir, expected', WINDOWS_TESTS) + def test_windows_on_posix(self, basedir, expected): + socketname = ipc._get_socketname_windows(basedir) + assert socketname == expected + @pytest.mark.mac @pytest.mark.parametrize('basedir, expected', POSIX_TESTS) def test_mac(self, basedir, expected): From 7ffe6a2c7817feb5e9817c6869849783539e60b2 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 22:12:17 +0200 Subject: [PATCH 218/337] Fix Python/PyQt casing --- FAQ.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FAQ.asciidoc b/FAQ.asciidoc index 8eba424de..9a540160a 100644 --- a/FAQ.asciidoc +++ b/FAQ.asciidoc @@ -173,7 +173,7 @@ What's the difference between insert and passthrough mode?:: Why takes it longer to open an URL in qutebrowser than in chromium?:: When opening an URL in an existing instance the normal qutebrowser - python script is started and a few PyQT libraries need to be + Python script is started and a few PyQt libraries need to be loaded until it is detected that there is an instance running where the URL is then passed to. This takes some time. One workaround is to use this From d895ad183d0542d624af93ab690c456a878178a1 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 22:12:32 +0200 Subject: [PATCH 219/337] Update authors --- README.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.asciidoc b/README.asciidoc index 972d03d51..f754ed8b4 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -165,10 +165,10 @@ Contributors, sorted by the number of commits in descending order: * Joel Torstensson * Jay Kamat * Patric Schmitz +* Fritz Reichwald * Tarcisio Fedrizzi * Claude * Philipp Hansch -* Fritz Reichwald * Corentin Julé * meles5 * Panagiotis Ktistakis From b3a9e09d6cc42675416f62e27a063569b90e760d Mon Sep 17 00:00:00 2001 From: Daniel Hahler Date: Sun, 18 Jun 2017 15:04:14 +0200 Subject: [PATCH 220/337] Add statusline widget for back/forward indicator Fixes https://github.com/qutebrowser/qutebrowser/issues/2737. --- qutebrowser/mainwindow/mainwindow.py | 2 + .../mainwindow/statusbar/backforward.py | 41 +++++++++++++ qutebrowser/mainwindow/statusbar/bar.py | 8 ++- scripts/dev/check_coverage.py | 2 + tests/helpers/stubs.py | 21 ++++++- .../mainwindow/statusbar/test_backforward.py | 57 +++++++++++++++++++ 6 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 qutebrowser/mainwindow/statusbar/backforward.py create mode 100644 tests/unit/mainwindow/statusbar/test_backforward.py diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index 2f43ba58e..3a7d78e97 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -473,6 +473,8 @@ class MainWindow(QWidget): tabs.tab_index_changed.connect(status.tabindex.on_tab_index_changed) tabs.cur_url_changed.connect(status.url.set_url) + tabs.cur_url_changed.connect(functools.partial( + status.backforward.on_tab_cur_url_changed, tabs=tabs)) tabs.cur_link_hovered.connect(status.url.set_hover_url) tabs.cur_load_status_changed.connect(status.url.on_load_status_changed) tabs.cur_fullscreen_requested.connect(self._on_fullscreen_requested) diff --git a/qutebrowser/mainwindow/statusbar/backforward.py b/qutebrowser/mainwindow/statusbar/backforward.py new file mode 100644 index 000000000..a58373122 --- /dev/null +++ b/qutebrowser/mainwindow/statusbar/backforward.py @@ -0,0 +1,41 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Navigation (back/forward) indicator displayed in the statusbar.""" + +from qutebrowser.mainwindow.statusbar import textbase + + +class Backforward(textbase.TextBase): + + """Shows navigation indicator (if you can go backward and/or forward).""" + + def on_tab_cur_url_changed(self, tabs): + """Called on URL changes.""" + tab = tabs.currentWidget() + if tab is None: # pragma: no cover + return + text = '' + if tab.history.can_go_back(): + text += '<' + if tab.history.can_go_forward(): + text += '>' + if text: + text = '[' + text + ']' + self.setText(text) diff --git a/qutebrowser/mainwindow/statusbar/bar.py b/qutebrowser/mainwindow/statusbar/bar.py index ee3665ce2..64a084327 100644 --- a/qutebrowser/mainwindow/statusbar/bar.py +++ b/qutebrowser/mainwindow/statusbar/bar.py @@ -25,8 +25,9 @@ from PyQt5.QtWidgets import QWidget, QHBoxLayout, QStackedLayout, QSizePolicy from qutebrowser.browser import browsertab from qutebrowser.config import config, style from qutebrowser.utils import usertypes, log, objreg, utils -from qutebrowser.mainwindow.statusbar import (command, progress, keystring, - percentage, url, tabindex) +from qutebrowser.mainwindow.statusbar import (backforward, command, progress, + keystring, percentage, url, + tabindex) from qutebrowser.mainwindow.statusbar import text as textwidget @@ -184,6 +185,9 @@ class StatusBar(QWidget): self.percentage = percentage.Percentage() self._hbox.addWidget(self.percentage) + self.backforward = backforward.Backforward() + self._hbox.addWidget(self.backforward) + self.tabindex = tabindex.TabIndex() self._hbox.addWidget(self.tabindex) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 7e5a2d47a..b03fb7ef4 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -117,6 +117,8 @@ PERFECT_FILES = [ 'mainwindow/statusbar/textbase.py'), ('tests/unit/mainwindow/statusbar/test_url.py', 'mainwindow/statusbar/url.py'), + ('tests/unit/mainwindow/statusbar/test_backforward.py', + 'mainwindow/statusbar/backforward.py'), ('tests/unit/mainwindow/test_messageview.py', 'mainwindow/messageview.py'), diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 0551f5a86..99cfbf01e 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -222,6 +222,23 @@ class FakeWebTabScroller(browsertab.AbstractScroller): return self._pos_perc +class FakeWebTabHistory(browsertab.AbstractHistory): + """Fake for Web{Kit,Engine}History.""" + + def __init__(self, tab, *, can_go_back, can_go_forward): + super().__init__(tab) + self._can_go_back = can_go_back + self._can_go_forward = can_go_forward + + def can_go_back(self): + assert self._can_go_back is not None + return self._can_go_back + + def can_go_forward(self): + assert self._can_go_forward is not None + return self._can_go_forward + + class FakeWebTab(browsertab.AbstractTab): """Fake AbstractTab to use in tests.""" @@ -229,12 +246,14 @@ class FakeWebTab(browsertab.AbstractTab): def __init__(self, url=FakeUrl(), title='', tab_id=0, *, scroll_pos_perc=(0, 0), load_status=usertypes.LoadStatus.success, - progress=0): + progress=0, can_go_back=None, can_go_forward=None): super().__init__(win_id=0, mode_manager=None, private=False) self._load_status = load_status self._title = title self._url = url self._progress = progress + self.history = FakeWebTabHistory(self, can_go_back=can_go_back, + can_go_forward=can_go_forward) self.scroller = FakeWebTabScroller(self, scroll_pos_perc) wrapped = QWidget() self._layout.wrap(self, wrapped) diff --git a/tests/unit/mainwindow/statusbar/test_backforward.py b/tests/unit/mainwindow/statusbar/test_backforward.py new file mode 100644 index 000000000..e29d592b2 --- /dev/null +++ b/tests/unit/mainwindow/statusbar/test_backforward.py @@ -0,0 +1,57 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + + +"""Test Progress widget.""" + +import pytest + +from qutebrowser.mainwindow.statusbar import backforward + + +@pytest.fixture +def backforward_widget(qtbot): + widget = backforward.Backforward() + qtbot.add_widget(widget) + return widget + + +@pytest.mark.parametrize('can_go_back, can_go_forward, expected_text', [ + (False, False, ''), + (True, False, '[<]'), + (False, True, '[>]'), + (True, True, '[<>]'), +]) +def test_backforward_widget(backforward_widget, stubs, + fake_web_tab, can_go_back, can_go_forward, + expected_text): + """Ensure the Backforward widget shows the correct text.""" + tab = fake_web_tab(can_go_back=can_go_back, can_go_forward=can_go_forward) + tabbed_browser = stubs.TabbedBrowserStub() + tabbed_browser.current_index = 1 + tabbed_browser.tabs = [tab] + backforward_widget.on_tab_cur_url_changed(tabbed_browser) + assert backforward_widget.text() == expected_text + + # Check that the widget gets reset if empty. + if can_go_back and can_go_forward: + tab = fake_web_tab(can_go_back=False, can_go_forward=False) + tabbed_browser.tabs = [tab] + backforward_widget.on_tab_cur_url_changed(tabbed_browser) + assert backforward_widget.text() == '' From b3b2f696735ff3fe9825af03a278f4ce13c4f63c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 23:09:50 +0200 Subject: [PATCH 221/337] Fix manifest --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 303c13db7..ec3e90473 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,7 +8,7 @@ graft icons graft doc/img graft misc/apparmor graft misc/userscripts -recursive-include scripts *.py +recursive-include scripts *.py *.sh include qutebrowser/utils/testfile include qutebrowser/git-commit-id include COPYING doc/* README.asciidoc CONTRIBUTING.asciidoc FAQ.asciidoc INSTALL.asciidoc CHANGELOG.asciidoc From 0e8175b8eb2a765ed93f3b68c7019a233c785c6a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 9 Jul 2017 23:27:34 +0200 Subject: [PATCH 222/337] Update docstrings/docs --- CHANGELOG.asciidoc | 5 +++++ README.asciidoc | 1 + tests/helpers/stubs.py | 1 + tests/unit/mainwindow/statusbar/test_backforward.py | 3 +-- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6b974e6ef..6471d0b9f 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -36,6 +36,11 @@ Major changes the entire browsing history. - Completely rewritten configuration system. +Added +~~~~~ + +- New back/forward indicator in the statusbar + Changed ~~~~~~~ diff --git a/README.asciidoc b/README.asciidoc index f754ed8b4..65a04a200 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -293,6 +293,7 @@ Contributors, sorted by the number of commits in descending order: * Derek Sivers * Daniel Lu * Daniel Jakots +* Daniel Hahler * Arseniy Seroka * Anton Grensjö * Andy Balaam diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 99cfbf01e..dc49d8fec 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -223,6 +223,7 @@ class FakeWebTabScroller(browsertab.AbstractScroller): class FakeWebTabHistory(browsertab.AbstractHistory): + """Fake for Web{Kit,Engine}History.""" def __init__(self, tab, *, can_go_back, can_go_forward): diff --git a/tests/unit/mainwindow/statusbar/test_backforward.py b/tests/unit/mainwindow/statusbar/test_backforward.py index e29d592b2..24c905c2c 100644 --- a/tests/unit/mainwindow/statusbar/test_backforward.py +++ b/tests/unit/mainwindow/statusbar/test_backforward.py @@ -17,8 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . - -"""Test Progress widget.""" +"""Test Backforward widget.""" import pytest From c6ed4fe4f9160d28702a3e4c9cf7fa2d3a1809ca Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Mon, 10 Jul 2017 00:28:47 +0200 Subject: [PATCH 223/337] Skip intermediate pages with :back/:forward and a count --- qutebrowser/browser/browsertab.py | 6 +++++ qutebrowser/browser/commands.py | 19 ++++++++------- qutebrowser/browser/webengine/webenginetab.py | 24 +++++++++++++++---- qutebrowser/browser/webkit/webkittab.py | 24 +++++++++++++++---- 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index b36b5d1c3..5ba0bd898 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -477,6 +477,12 @@ class AbstractHistory: def can_go_forward(self): raise NotImplementedError + def itemAt(self, i): + raise NotImplementedError + + def goToItem(self, item): + raise NotImplementedError + def serialize(self): """Serialize into an opaque format understood by self.deserialize.""" raise NotImplementedError diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index f0c31aa76..e11b9a61b 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -540,15 +540,16 @@ class CommandDispatcher: else: widget = self._current_widget() - for _ in range(count): - if forward: - if not widget.history.can_go_forward(): - raise cmdexc.CommandError("At end of history.") - widget.history.forward() - else: - if not widget.history.can_go_back(): - raise cmdexc.CommandError("At beginning of history.") - widget.history.back() + if forward: + try: + widget.history.forward(count) + except IndexError: + raise cmdexc.CommandError("At end of history.") + else: + try: + widget.history.back(count) + except IndexError: + raise cmdexc.CommandError("At beginning of history.") @cmdutils.register(instance='command-dispatcher', scope='window') @cmdutils.argument('count', count=True) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 11672d3b3..a86b9c6c1 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -411,11 +411,21 @@ class WebEngineHistory(browsertab.AbstractHistory): def current_idx(self): return self._history.currentItemIndex() - def back(self): - self._history.back() + def back(self, count): + idx = self.current_idx() - count + if idx >= 0: + self.goToItem(self.itemAt(idx)) + else: + self.goToItem(self.itemAt(0)) + raise IndexError - def forward(self): - self._history.forward() + def forward(self, count): + idx = self.current_idx() + count + if idx < len(self): + self.goToItem(self.itemAt(idx)) + else: + self.goToItem(self.itemAt(len(self) - 1)) + raise IndexError def can_go_back(self): return self._history.canGoBack() @@ -423,6 +433,12 @@ class WebEngineHistory(browsertab.AbstractHistory): def can_go_forward(self): return self._history.canGoForward() + def itemAt(self, i): + return self._history.itemAt(i) + + def goToItem(self, item): + return self._history.goToItem(item) + def serialize(self): if not qtutils.version_check('5.9'): # WORKAROUND for diff --git a/qutebrowser/browser/webkit/webkittab.py b/qutebrowser/browser/webkit/webkittab.py index c29aa15cb..6306ac542 100644 --- a/qutebrowser/browser/webkit/webkittab.py +++ b/qutebrowser/browser/webkit/webkittab.py @@ -506,11 +506,21 @@ class WebKitHistory(browsertab.AbstractHistory): def current_idx(self): return self._history.currentItemIndex() - def back(self): - self._history.back() + def back(self, count): + idx = self.current_idx() - count + if idx >= 0: + self.goToItem(self.itemAt(idx)) + else: + self.goToItem(self.itemAt(0)) + raise IndexError - def forward(self): - self._history.forward() + def forward(self, count): + idx = self.current_idx() + count + if idx < len(self): + self.goToItem(self.itemAt(idx)) + else: + self.goToItem(self.itemAt(len(self) - 1)) + raise IndexError def can_go_back(self): return self._history.canGoBack() @@ -518,6 +528,12 @@ class WebKitHistory(browsertab.AbstractHistory): def can_go_forward(self): return self._history.canGoForward() + def itemAt(self, i): + return self._history.itemAt(i) + + def goToItem(self, item): + return self._history.goToItem(item) + def serialize(self): return qtutils.serialize(self._history) From bf074d14de2e51f4ea925f788834dd02a49bc979 Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Mon, 10 Jul 2017 01:00:48 +0200 Subject: [PATCH 224/337] Adjust back/forward method arguments in AbstractHistory class --- qutebrowser/browser/browsertab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index 5ba0bd898..ab26878e3 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -465,10 +465,10 @@ class AbstractHistory: def current_idx(self): raise NotImplementedError - def back(self): + def back(self, count): raise NotImplementedError - def forward(self): + def forward(self, count): raise NotImplementedError def can_go_back(self): From 5fb6cb713b7047cec245ea7a3a19dddfbc1cf5aa Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 10 Jul 2017 07:59:56 +0200 Subject: [PATCH 225/337] Hide back/forward widget when there's no text --- qutebrowser/mainwindow/statusbar/backforward.py | 1 + tests/unit/mainwindow/statusbar/test_backforward.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/qutebrowser/mainwindow/statusbar/backforward.py b/qutebrowser/mainwindow/statusbar/backforward.py index a58373122..9ed18ffb0 100644 --- a/qutebrowser/mainwindow/statusbar/backforward.py +++ b/qutebrowser/mainwindow/statusbar/backforward.py @@ -39,3 +39,4 @@ class Backforward(textbase.TextBase): if text: text = '[' + text + ']' self.setText(text) + self.setVisible(bool(text)) diff --git a/tests/unit/mainwindow/statusbar/test_backforward.py b/tests/unit/mainwindow/statusbar/test_backforward.py index 24c905c2c..bf37cd55f 100644 --- a/tests/unit/mainwindow/statusbar/test_backforward.py +++ b/tests/unit/mainwindow/statusbar/test_backforward.py @@ -47,6 +47,7 @@ def test_backforward_widget(backforward_widget, stubs, tabbed_browser.tabs = [tab] backforward_widget.on_tab_cur_url_changed(tabbed_browser) assert backforward_widget.text() == expected_text + assert backforward_widget.isVisible() == bool(expected_text) # Check that the widget gets reset if empty. if can_go_back and can_go_forward: @@ -54,3 +55,4 @@ def test_backforward_widget(backforward_widget, stubs, tabbed_browser.tabs = [tab] backforward_widget.on_tab_cur_url_changed(tabbed_browser) assert backforward_widget.text() == '' + assert not backforward_widget.isVisible() From e81dcccacef3c241361a6bcf98e3038ec9bd53c0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 10 Jul 2017 08:00:09 +0200 Subject: [PATCH 226/337] Add a test for a None currentWidget with backforward widget --- .../mainwindow/statusbar/backforward.py | 4 +++- tests/helpers/stubs.py | 5 ++++- .../mainwindow/statusbar/test_backforward.py | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/qutebrowser/mainwindow/statusbar/backforward.py b/qutebrowser/mainwindow/statusbar/backforward.py index 9ed18ffb0..233cb53f8 100644 --- a/qutebrowser/mainwindow/statusbar/backforward.py +++ b/qutebrowser/mainwindow/statusbar/backforward.py @@ -29,7 +29,9 @@ class Backforward(textbase.TextBase): def on_tab_cur_url_changed(self, tabs): """Called on URL changes.""" tab = tabs.currentWidget() - if tab is None: # pragma: no cover + if tab is None: + self.setText('') + self.hide() return text = '' if tab.history.can_go_back(): diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index dc49d8fec..368fe33e0 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -620,7 +620,10 @@ class TabbedBrowserStub(QObject): return self.current_index def currentWidget(self): - return self.tabs[self.currentIndex() - 1] + idx = self.currentIndex() + if idx == -1: + return None + return self.tabs[idx - 1] def tabopen(self, url): self.opened_url = url diff --git a/tests/unit/mainwindow/statusbar/test_backforward.py b/tests/unit/mainwindow/statusbar/test_backforward.py index bf37cd55f..f2dec3d3f 100644 --- a/tests/unit/mainwindow/statusbar/test_backforward.py +++ b/tests/unit/mainwindow/statusbar/test_backforward.py @@ -56,3 +56,21 @@ def test_backforward_widget(backforward_widget, stubs, backforward_widget.on_tab_cur_url_changed(tabbed_browser) assert backforward_widget.text() == '' assert not backforward_widget.isVisible() + + +def test_none_tab(backforward_widget, stubs, fake_web_tab): + """Make sure nothing crashes when passing None as tab.""" + tab = fake_web_tab(can_go_back=True, can_go_forward=True) + tabbed_browser = stubs.TabbedBrowserStub() + tabbed_browser.current_index = 1 + tabbed_browser.tabs = [tab] + backforward_widget.on_tab_cur_url_changed(tabbed_browser) + + assert backforward_widget.text() == '[<>]' + assert backforward_widget.isVisible() + + tabbed_browser.current_index = -1 + backforward_widget.on_tab_cur_url_changed(tabbed_browser) + + assert backforward_widget.text() == '' + assert not backforward_widget.isVisible() From 3c1b05c81e8ce00ed647001f7623c79f2612b770 Mon Sep 17 00:00:00 2001 From: Yashar Shahi Date: Mon, 10 Jul 2017 18:05:35 +0430 Subject: [PATCH 227/337] Show messages longer if there are multiple of them --- qutebrowser/mainwindow/messageview.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qutebrowser/mainwindow/messageview.py b/qutebrowser/mainwindow/messageview.py index 7d0d2b682..382853d11 100644 --- a/qutebrowser/mainwindow/messageview.py +++ b/qutebrowser/mainwindow/messageview.py @@ -100,6 +100,10 @@ class MessageView(QWidget): """Configure self._clear_timer according to the config.""" interval = config.get('ui', 'message-timeout') if interval != 0: + multimpy_by = len(self._messages) + 1 + if multimpy_by > 5: + multimpy_by = 5 + interval *= multimpy_by self._clear_timer.setInterval(interval) @pyqtSlot() @@ -128,6 +132,7 @@ class MessageView(QWidget): self._vbox.addWidget(widget) widget.show() if config.get('ui', 'message-timeout') != 0: + self._set_clear_timer_interval() self._clear_timer.start() self._messages.append(widget) self._last_text = text From 6ab49fdf1dfe0589dea4ff7d892b2da476f0dab2 Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Mon, 10 Jul 2017 15:43:35 +0200 Subject: [PATCH 228/337] Move back/forward logic to AbstractHistory, fix method names --- qutebrowser/browser/browsertab.py | 18 +++++++++++++---- qutebrowser/browser/commands.py | 8 ++++---- qutebrowser/browser/webengine/webenginetab.py | 20 ++----------------- qutebrowser/browser/webkit/webkittab.py | 20 ++----------------- 4 files changed, 22 insertions(+), 44 deletions(-) diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index ab26878e3..e935589a7 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -466,10 +466,20 @@ class AbstractHistory: raise NotImplementedError def back(self, count): - raise NotImplementedError + idx = self.current_idx() - count + if idx >= 0: + self._go_to_item(self._item_at(idx)) + else: + self._go_to_item(self._item_at(0)) + raise WebTabError("At beginning of history.") def forward(self, count): - raise NotImplementedError + idx = self.current_idx() + count + if idx < len(self): + self._go_to_item(self._item_at(idx)) + else: + self._go_to_item(self._item_at(len(self) - 1)) + raise WebTabError("At end of history.") def can_go_back(self): raise NotImplementedError @@ -477,10 +487,10 @@ class AbstractHistory: def can_go_forward(self): raise NotImplementedError - def itemAt(self, i): + def _item_at(self, i): raise NotImplementedError - def goToItem(self, item): + def _go_to_item(self, item): raise NotImplementedError def serialize(self): diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index e11b9a61b..682c8e3c7 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -543,13 +543,13 @@ class CommandDispatcher: if forward: try: widget.history.forward(count) - except IndexError: - raise cmdexc.CommandError("At end of history.") + except browsertab.WebTabError as e: + raise cmdexc.CommandError(e) else: try: widget.history.back(count) - except IndexError: - raise cmdexc.CommandError("At beginning of history.") + except browsertab.WebTabError as e: + raise cmdexc.CommandError(e) @cmdutils.register(instance='command-dispatcher', scope='window') @cmdutils.argument('count', count=True) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index a86b9c6c1..8d328f5e8 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -411,32 +411,16 @@ class WebEngineHistory(browsertab.AbstractHistory): def current_idx(self): return self._history.currentItemIndex() - def back(self, count): - idx = self.current_idx() - count - if idx >= 0: - self.goToItem(self.itemAt(idx)) - else: - self.goToItem(self.itemAt(0)) - raise IndexError - - def forward(self, count): - idx = self.current_idx() + count - if idx < len(self): - self.goToItem(self.itemAt(idx)) - else: - self.goToItem(self.itemAt(len(self) - 1)) - raise IndexError - def can_go_back(self): return self._history.canGoBack() def can_go_forward(self): return self._history.canGoForward() - def itemAt(self, i): + def _item_at(self, i): return self._history.itemAt(i) - def goToItem(self, item): + def _go_to_item(self, item): return self._history.goToItem(item) def serialize(self): diff --git a/qutebrowser/browser/webkit/webkittab.py b/qutebrowser/browser/webkit/webkittab.py index 6306ac542..348d2c628 100644 --- a/qutebrowser/browser/webkit/webkittab.py +++ b/qutebrowser/browser/webkit/webkittab.py @@ -506,32 +506,16 @@ class WebKitHistory(browsertab.AbstractHistory): def current_idx(self): return self._history.currentItemIndex() - def back(self, count): - idx = self.current_idx() - count - if idx >= 0: - self.goToItem(self.itemAt(idx)) - else: - self.goToItem(self.itemAt(0)) - raise IndexError - - def forward(self, count): - idx = self.current_idx() + count - if idx < len(self): - self.goToItem(self.itemAt(idx)) - else: - self.goToItem(self.itemAt(len(self) - 1)) - raise IndexError - def can_go_back(self): return self._history.canGoBack() def can_go_forward(self): return self._history.canGoForward() - def itemAt(self, i): + def _item_at(self, i): return self._history.itemAt(i) - def goToItem(self, item): + def _go_to_item(self, item): return self._history.goToItem(item) def serialize(self): From 9e6b84e31eaffa72dcb1e11a0c4e331d00e36814 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Jul 2017 15:55:12 +0200 Subject: [PATCH 229/337] Update flake8-tidy-imports from 1.0.6 to 1.1.0 --- misc/requirements/requirements-flake8.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index ecec607b5..f68bcb227 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -11,7 +11,7 @@ flake8-pep3101==1.0 # rq.filter: < 1.1 flake8-polyfill==1.0.1 flake8-putty==0.4.0 flake8-string-format==0.2.3 -flake8-tidy-imports==1.0.6 +flake8-tidy-imports==1.1.0 flake8-tuple==0.2.13 mccabe==0.6.1 packaging==16.8 From 9da52c5d865f7256eb1bb76dea0c96390123e807 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Jul 2017 15:55:14 +0200 Subject: [PATCH 230/337] Update hypothesis from 3.11.6 to 3.12.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 57a4daff7..5993099c6 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -12,7 +12,7 @@ Flask==0.12.2 glob2==0.5 httpbin==0.5.0 hunter==1.4.1 -hypothesis==3.11.6 +hypothesis==3.12.0 itsdangerous==0.24 # Jinja2==2.9.6 Mako==1.0.6 From 57caf80e5d3ff72980c394fef29661509e86e6c5 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Jul 2017 15:55:15 +0200 Subject: [PATCH 231/337] Update pytest from 3.1.2 to 3.1.3 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 5993099c6..81aa169df 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -20,7 +20,7 @@ Mako==1.0.6 parse==1.8.2 parse-type==0.3.4 py==1.4.34 -pytest==3.1.2 +pytest==3.1.3 pytest-bdd==2.18.2 pytest-benchmark==3.0.0 pytest-catchlog==1.2.2 From 44270b37b9f05fce6659c5feee8fc3f5cfe255c6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Jul 2017 15:55:17 +0200 Subject: [PATCH 232/337] Update vulture from 0.14 to 0.15 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 81aa169df..40de7a673 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -35,5 +35,5 @@ pytest-travis-fold==1.2.0 pytest-xvfb==1.0.0 PyVirtualDisplay==0.2.1 six==1.10.0 -vulture==0.14 +vulture==0.15 Werkzeug==0.12.2 From 34eddc92ff4082f0071fdf7b3aed53fa7e97e5a0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 10 Jul 2017 15:55:18 +0200 Subject: [PATCH 233/337] Update vulture from 0.14 to 0.15 --- misc/requirements/requirements-vulture.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-vulture.txt b/misc/requirements/requirements-vulture.txt index 56e20c603..9edb87f9c 100644 --- a/misc/requirements/requirements-vulture.txt +++ b/misc/requirements/requirements-vulture.txt @@ -1,3 +1,3 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -vulture==0.14 +vulture==0.15 From 9c83ea4717e63c3ea71b3f11039cfb8122a29840 Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Mon, 10 Jul 2017 15:58:11 +0200 Subject: [PATCH 234/337] Refactor _back_forward --- qutebrowser/browser/commands.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 682c8e3c7..8dde90602 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -540,16 +540,13 @@ class CommandDispatcher: else: widget = self._current_widget() - if forward: - try: + try: + if forward: widget.history.forward(count) - except browsertab.WebTabError as e: - raise cmdexc.CommandError(e) - else: - try: + else: widget.history.back(count) - except browsertab.WebTabError as e: - raise cmdexc.CommandError(e) + except browsertab.WebTabError as e: + raise cmdexc.CommandError(e) @cmdutils.register(instance='command-dispatcher', scope='window') @cmdutils.argument('count', count=True) From cf2f81aae1302f7ed38ad425fb77fa07ea29a803 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 10 Jul 2017 15:59:05 +0200 Subject: [PATCH 235/337] Remove myself from CODEOWNERS I watch the repo anyways --- .github/CODEOWNERS | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f623e0a93..2b8c12de9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,10 +1,8 @@ -* @The-Compiler - -qutebrowser/browser/history.py @The-Compiler @rcorre -qutebrowser/completion/* @The-Compiler @rcorre -qutebrowser/misc/sql.py @The-Compiler @rcorre -tests/end2end/features/completion.feature @The-Compiler @rcorre -tests/end2end/features/test_completion_bdd.py @The-Compiler @rcorre -tests/unit/browser/test_history.py @The-Compiler @rcorre -tests/unit/completion/* @The-Compiler @rcorre -tests/unit/misc/test_sql.py @The-Compiler @rcorre +qutebrowser/browser/history.py @rcorre +qutebrowser/completion/* @rcorre +qutebrowser/misc/sql.py @rcorre +tests/end2end/features/completion.feature @rcorre +tests/end2end/features/test_completion_bdd.py @rcorre +tests/unit/browser/test_history.py @rcorre +tests/unit/completion/* @rcorre +tests/unit/misc/test_sql.py @rcorre From 03c70f0421c728cd6be3c639d92d4d8161bd00d6 Mon Sep 17 00:00:00 2001 From: Yashar Shahi Date: Mon, 10 Jul 2017 18:58:45 +0430 Subject: [PATCH 236/337] An empty file to trigger travis. --- trigger_travis | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 trigger_travis diff --git a/trigger_travis b/trigger_travis new file mode 100644 index 000000000..e69de29bb From c015e9cc5df6e8eb11efa44c733e4b5be93f5081 Mon Sep 17 00:00:00 2001 From: Yashar Shahi Date: Mon, 10 Jul 2017 19:20:32 +0430 Subject: [PATCH 237/337] Revert "An empty file to trigger travis." This reverts commit 03c70f0421c728cd6be3c639d92d4d8161bd00d6. --- trigger_travis | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 trigger_travis diff --git a/trigger_travis b/trigger_travis deleted file mode 100644 index e69de29bb..000000000 From cb0bd2c52d93512e81cb41199bea1394833c7729 Mon Sep 17 00:00:00 2001 From: Yashar Shahi Date: Mon, 10 Jul 2017 19:21:35 +0430 Subject: [PATCH 238/337] Do not call _set_timer_interval() at constructor No need to call _set_timer_interval() at constructor since it's called every time timer is going to be started. --- qutebrowser/mainwindow/messageview.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qutebrowser/mainwindow/messageview.py b/qutebrowser/mainwindow/messageview.py index 382853d11..04da214f7 100644 --- a/qutebrowser/mainwindow/messageview.py +++ b/qutebrowser/mainwindow/messageview.py @@ -83,7 +83,6 @@ class MessageView(QWidget): self._clear_timer = QTimer() self._clear_timer.timeout.connect(self.clear_messages) - self._set_clear_timer_interval() objreg.get('config').changed.connect(self._set_clear_timer_interval) self._last_text = None From 045831f3c7f698c98e75a215e41b4062029aba95 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 10 Jul 2017 16:57:26 +0200 Subject: [PATCH 239/337] Fix coverage check --- qutebrowser/mainwindow/statusbar/backforward.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/statusbar/backforward.py b/qutebrowser/mainwindow/statusbar/backforward.py index 233cb53f8..302de4d95 100644 --- a/qutebrowser/mainwindow/statusbar/backforward.py +++ b/qutebrowser/mainwindow/statusbar/backforward.py @@ -29,7 +29,8 @@ class Backforward(textbase.TextBase): def on_tab_cur_url_changed(self, tabs): """Called on URL changes.""" tab = tabs.currentWidget() - if tab is None: + if tab is None: # pragma: no cover + # WORKAROUND: Doesn't get tested on older PyQt self.setText('') self.hide() return From a631c971d98784b2cbd61c5b73900802ddfec54e Mon Sep 17 00:00:00 2001 From: Yashar Shahi Date: Mon, 10 Jul 2017 20:52:42 +0430 Subject: [PATCH 240/337] Add tests for show messages longer Add tests for "Show messages longer if there are multiple of them." --- tests/unit/mainwindow/test_messageview.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/mainwindow/test_messageview.py b/tests/unit/mainwindow/test_messageview.py index ebecc9398..7790d343e 100644 --- a/tests/unit/mainwindow/test_messageview.py +++ b/tests/unit/mainwindow/test_messageview.py @@ -104,6 +104,14 @@ def test_changing_timer_with_messages_shown(qtbot, view, config_stub): config_stub.set('ui', 'message-timeout', 100) +@pytest.mark.parametrize('count', [1, 2, 3, 4, 5, 6, 7]) +def test_show_messages_longer_when_there_are_multiple_of_them(view, config_stub, count): + """When there are multiple messages, messages should be shown longer than usual. but with an upper maximum of 5""" + for message_number in range(1, count+1): + view.show_message(usertypes.MessageLevel.info, 'test ' + str(message_number)) + assert view._clear_timer.interval() == min(5, count) * config_stub['ui']['message-timeout'] + + @pytest.mark.parametrize('replace1, replace2, length', [ (False, False, 2), # Two stacked messages (True, True, 1), # Two replaceable messages From 196f4a67b2b862647e81ca6662626e74eec7961d Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 10 Jul 2017 18:34:48 +0200 Subject: [PATCH 241/337] Update docs --- CHANGELOG.asciidoc | 1 + README.asciidoc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6471d0b9f..faff46d36 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -47,6 +47,7 @@ Changed - Upgrading qutebrowser with a version older than v0.4.0 still running now won't work properly anymore. - Using `:download` now uses the page's title as filename. +- Using `:back` or `:forward` with a count now skips intermediate pages. Fixes ~~~~~ diff --git a/README.asciidoc b/README.asciidoc index 65a04a200..825c6fd6b 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -177,11 +177,11 @@ Contributors, sorted by the number of commits in descending order: * Thorsten Wißmann * Austin Anderson * Jimmy +* Iordanis Grigoriou * Niklas Haas * Maciej Wołczyk * Clayton Craft * sandrosc -* Iordanis Grigoriou * Alexey "Averrin" Nabrodov * pkill9 * nanjekyejoannah From 1cb23f1193e1664bb0e2de6e266f20d48b9de148 Mon Sep 17 00:00:00 2001 From: Yashar Shahi Date: Mon, 10 Jul 2017 21:11:38 +0430 Subject: [PATCH 242/337] Change timer interval after appending to _messages --- qutebrowser/mainwindow/messageview.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/qutebrowser/mainwindow/messageview.py b/qutebrowser/mainwindow/messageview.py index 04da214f7..36c778d12 100644 --- a/qutebrowser/mainwindow/messageview.py +++ b/qutebrowser/mainwindow/messageview.py @@ -99,10 +99,7 @@ class MessageView(QWidget): """Configure self._clear_timer according to the config.""" interval = config.get('ui', 'message-timeout') if interval != 0: - multimpy_by = len(self._messages) + 1 - if multimpy_by > 5: - multimpy_by = 5 - interval *= multimpy_by + interval *= min(5, len(self._messages)) self._clear_timer.setInterval(interval) @pyqtSlot() @@ -130,13 +127,13 @@ class MessageView(QWidget): widget = Message(level, text, replace=replace, parent=self) self._vbox.addWidget(widget) widget.show() - if config.get('ui', 'message-timeout') != 0: - self._set_clear_timer_interval() - self._clear_timer.start() self._messages.append(widget) self._last_text = text self.show() self.update_geometry.emit() + if config.get('ui', 'message-timeout') != 0: + self._set_clear_timer_interval() + self._clear_timer.start() def mousePressEvent(self, e): """Clear messages when they are clicked on.""" From 7da690885070fcc3708319ce52a07e6a194061fe Mon Sep 17 00:00:00 2001 From: Yashar Shahi Date: Mon, 10 Jul 2017 21:14:55 +0430 Subject: [PATCH 243/337] Check for interval being positive. Check for interval being positive instead of checking for it to be non-zero. So if somehow some unexpected thing happend and made message-timeout negative, the bug doesn't cascade. --- qutebrowser/mainwindow/messageview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/messageview.py b/qutebrowser/mainwindow/messageview.py index 36c778d12..657835dba 100644 --- a/qutebrowser/mainwindow/messageview.py +++ b/qutebrowser/mainwindow/messageview.py @@ -98,7 +98,7 @@ class MessageView(QWidget): def _set_clear_timer_interval(self): """Configure self._clear_timer according to the config.""" interval = config.get('ui', 'message-timeout') - if interval != 0: + if interval > 0: interval *= min(5, len(self._messages)) self._clear_timer.setInterval(interval) From 882dc7553655f067e75c6f952c12d2c1f81d2d67 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 11 Jul 2017 08:38:06 +0200 Subject: [PATCH 244/337] Set default count for AbstractHistory.back/.forward Otherwise, using back/forward mouse buttons will crash. --- qutebrowser/browser/browsertab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index e935589a7..b94172118 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -465,7 +465,7 @@ class AbstractHistory: def current_idx(self): raise NotImplementedError - def back(self, count): + def back(self, count=1): idx = self.current_idx() - count if idx >= 0: self._go_to_item(self._item_at(idx)) @@ -473,7 +473,7 @@ class AbstractHistory: self._go_to_item(self._item_at(0)) raise WebTabError("At beginning of history.") - def forward(self, count): + def forward(self, count=1): idx = self.current_idx() + count if idx < len(self): self._go_to_item(self._item_at(idx)) From 1e58c87380b3140240fd207d0742d63f57698238 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 11 Jul 2017 09:25:53 +0200 Subject: [PATCH 245/337] Improve test for messageview timeout --- tests/unit/mainwindow/test_messageview.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/unit/mainwindow/test_messageview.py b/tests/unit/mainwindow/test_messageview.py index 7790d343e..782e38ae5 100644 --- a/tests/unit/mainwindow/test_messageview.py +++ b/tests/unit/mainwindow/test_messageview.py @@ -104,12 +104,17 @@ def test_changing_timer_with_messages_shown(qtbot, view, config_stub): config_stub.set('ui', 'message-timeout', 100) -@pytest.mark.parametrize('count', [1, 2, 3, 4, 5, 6, 7]) -def test_show_messages_longer_when_there_are_multiple_of_them(view, config_stub, count): - """When there are multiple messages, messages should be shown longer than usual. but with an upper maximum of 5""" +@pytest.mark.parametrize('count, expected', [(1, 100), (3, 300), + (5, 500), (7, 500)]) +def test_show_multiple_messages_longer(view, count, expected): + """When there are multiple messages, messages should be shown longer. + + There is an upper maximum to avoid messages never disappearing. + """ for message_number in range(1, count+1): - view.show_message(usertypes.MessageLevel.info, 'test ' + str(message_number)) - assert view._clear_timer.interval() == min(5, count) * config_stub['ui']['message-timeout'] + view.show_message(usertypes.MessageLevel.info, + 'test ' + str(message_number)) + assert view._clear_timer.interval() == expected @pytest.mark.parametrize('replace1, replace2, length', [ From 6f930be08e763b72b002ed8e53cff1e583238929 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 11 Jul 2017 09:26:36 +0200 Subject: [PATCH 246/337] Update docs --- CHANGELOG.asciidoc | 1 + README.asciidoc | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6471d0b9f..6dd448d67 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -47,6 +47,7 @@ Changed - Upgrading qutebrowser with a version older than v0.4.0 still running now won't work properly anymore. - Using `:download` now uses the page's title as filename. +- When there are multiple messages shown, the timeout is increased. Fixes ~~~~~ diff --git a/README.asciidoc b/README.asciidoc index 65a04a200..b2cc291f5 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -192,6 +192,7 @@ Contributors, sorted by the number of commits in descending order: * Peter Vilim * Jacob Sword * knaggita +* Yashar Shahi * Oliver Caldwell * Nikolay Amiantov * Marius From 3dfa36fad1a393fbba45bddf141ca11e736a7170 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 11 Jul 2017 12:59:44 +0200 Subject: [PATCH 247/337] Update changelog --- CHANGELOG.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index ab2ecc2a7..8268f1c0a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -14,7 +14,7 @@ This project adheres to http://semver.org/[Semantic Versioning]. // `Fixed` for any bug fixes. // `Security` to invite users to upgrade in case of vulnerabilities. -v0.1.0 (unreleased) +v1.0.0 (unreleased) ------------------- Breaking changes From 182d067ff8f09de1f92461e5d87011055625eb1d Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 7 Jul 2017 21:16:50 -0400 Subject: [PATCH 248/337] SQL code review fixes. - Fix comment and empty line check in _parse_entry - connect layoutAboutToBeChanged signal - assert sort_order is None if sort_by is None - modify sql init failure message to ask about Qt sqlite support. --- qutebrowser/completion/models/completionmodel.py | 1 + qutebrowser/completion/models/sqlcategory.py | 2 ++ qutebrowser/misc/sql.py | 3 ++- tests/unit/completion/test_completionmodel.py | 9 ++++++--- tests/unit/completion/test_sqlcategory.py | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 778e0cac9..3e48076e5 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -59,6 +59,7 @@ class CompletionModel(QAbstractItemModel): def add_category(self, cat): """Add a completion category to the model.""" self._categories.append(cat) + cat.layoutAboutToBeChanged.connect(self.layoutAboutToBeChanged) cat.layoutChanged.connect(self.layoutChanged) def data(self, index, role=Qt.DisplayRole): diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/sqlcategory.py index 4819e940b..2664edbfe 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/sqlcategory.py @@ -59,6 +59,8 @@ class SqlCategory(QSqlQueryModel): if sort_by: assert sort_order in ['asc', 'desc'], sort_order querystr += ' order by {} {}'.format(sort_by, sort_order) + else: + assert sort_order is None, sort_order self._query = sql.Query(querystr, forward_only=False) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 4ae37f01f..a288df475 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -38,7 +38,8 @@ def init(db_path): """Initialize the SQL database connection.""" database = QSqlDatabase.addDatabase('QSQLITE') if not database.isValid(): - raise SqlException('Failed to add database. Is sqlite installed?') + raise SqlException('Failed to add database. ' + 'Are sqlite and Qt sqlite support installed?') database.setDatabaseName(db_path) if not database.open(): raise SqlException("Failed to open sqlite database at {}: {}" diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index ce0fe3765..4d1d3f123 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -36,7 +36,7 @@ def test_first_last_item(counts): """Test that first() and last() index to the first and last items.""" model = completionmodel.CompletionModel() for c in counts: - cat = mock.Mock(spec=['layoutChanged']) + cat = mock.Mock(spec=['layoutChanged', 'layoutAboutToBeChanged']) cat.rowCount = mock.Mock(return_value=c, spec=[]) model.add_category(cat) data = [i for i, rowCount in enumerate(counts) if rowCount > 0] @@ -60,7 +60,8 @@ def test_first_last_item(counts): def test_count(counts): model = completionmodel.CompletionModel() for c in counts: - cat = mock.Mock(spec=['rowCount', 'layoutChanged']) + cat = mock.Mock(spec=['rowCount', 'layoutChanged', + 'layoutAboutToBeChanged']) cat.rowCount = mock.Mock(return_value=c, spec=[]) model.add_category(cat) assert model.count() == sum(counts) @@ -70,7 +71,9 @@ def test_count(counts): def test_set_pattern(pat): """Validate the filtering and sorting results of set_pattern.""" model = completionmodel.CompletionModel() - cats = [mock.Mock(spec=['set_pattern', 'layoutChanged']) for _ in range(3)] + cats = [mock.Mock(spec=['set_pattern', 'layoutChanged', + 'layoutAboutToBeChanged']) + for _ in range(3)] for c in cats: c.set_pattern = mock.Mock(spec=[]) model.add_category(c) diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py index 851bad257..a9321e33e 100644 --- a/tests/unit/completion/test_sqlcategory.py +++ b/tests/unit/completion/test_sqlcategory.py @@ -32,7 +32,7 @@ pytestmark = pytest.mark.usefixtures('init_sql') @pytest.mark.parametrize('sort_by, sort_order, data, expected', [ - (None, 'asc', + (None, None, [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), From 53620ecce4db7c92d971d12147e3d178fe9d175f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 11 Jul 2017 23:41:08 +0200 Subject: [PATCH 249/337] Fix printing on macOS Fixes #2798 --- CHANGELOG.asciidoc | 1 + qutebrowser/browser/commands.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 8268f1c0a..d3bc99b2a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -64,6 +64,7 @@ Fixes - Fixed empty space being shown after tabs in the tabbar in some cases. - Fixed `:restart` in private browsing mode. +- Fixed printing on macOS. v0.11.0 ------- diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 8dde90602..3f1a54a6f 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -20,11 +20,12 @@ """Command dispatcher for TabbedBrowser.""" import os +import sys import os.path import shlex import functools -from PyQt5.QtWidgets import QApplication, QTabBar +from PyQt5.QtWidgets import QApplication, QTabBar, QDialog from PyQt5.QtCore import Qt, QUrl, QEvent, QUrlQuery from PyQt5.QtGui import QKeyEvent from PyQt5.QtPrintSupport import QPrintDialog, QPrintPreviewDialog @@ -436,9 +437,18 @@ class CommandDispatcher: message.error("Printing failed!") diag.deleteLater() + def do_print(): + """Called when the dialog was closed.""" + tab.printing.to_printer(diag.printer(), print_callback) + diag = QPrintDialog(tab) - diag.open(lambda: tab.printing.to_printer(diag.printer(), - print_callback)) + if sys.platform == 'darwin': + # For some reason we get a segfault when using open() on macOS + ret = diag.exec_() + if ret == QDialog.Accepted: + do_print() + else: + diag.open(do_print) @cmdutils.register(instance='command-dispatcher', name='print', scope='window') From ea459a1eca237955cc004da22145a0e6b598a6bf Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 12 Jul 2017 08:19:31 -0400 Subject: [PATCH 250/337] SQL code review fixes. - Ignore invalid variable name in flake8 (pylint already checks this and we don't want to have to double-ignore) - Fix and test completion bug with `:set asdf ` - Remove unused import - Use `assert not func.called` instead of `func.assert_not_called` for backwards compatibility --- .flake8 | 3 ++- qutebrowser/completion/completionwidget.py | 2 ++ tests/helpers/fixtures.py | 1 + tests/unit/browser/test_history.py | 1 - tests/unit/completion/test_completionwidget.py | 7 ++++++- 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.flake8 b/.flake8 index d967a505b..b87fef8b2 100644 --- a/.flake8 +++ b/.flake8 @@ -11,6 +11,7 @@ exclude = .*,__pycache__,resources.py # (for pytest's __tracebackhide__) # F401: Unused import # N802: function name should be lowercase +# N806: variable in function should be lowercase # P101: format string does contain unindexed parameters # P102: docstring does contain unindexed parameters # P103: other string does contain unindexed parameters @@ -27,7 +28,7 @@ exclude = .*,__pycache__,resources.py ignore = E128,E226,E265,E501,E402,E266,E722,E731, F401, - N802, + N802,N806 P101,P102,P103, D102,D103,D104,D105,D209,D211,D402,D403 min-version = 3.4.0 diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 9b8d31ad6..68466630a 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -290,6 +290,8 @@ class CompletionView(QTreeView): self.expand(model.index(i, 0)) def set_pattern(self, pattern): + if not self.model(): + return self.pattern = pattern with debug.log_time(log.completion, 'Set pattern {}'.format(pattern)): self.model().set_pattern(pattern) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index e38a3ce49..74c63f251 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -490,6 +490,7 @@ class ModelValidator: def __init__(self, modeltester): modeltester.data_display_may_return_none = True + self._model = None self._modeltester = modeltester def set_model(self, model): diff --git a/tests/unit/browser/test_history.py b/tests/unit/browser/test_history.py index 0b5e6eb41..5a8425646 100644 --- a/tests/unit/browser/test_history.py +++ b/tests/unit/browser/test_history.py @@ -20,7 +20,6 @@ """Tests for the global page history.""" import logging -import os import pytest from PyQt5.QtCore import QUrl diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index cad45b5c1..207e557a8 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -89,6 +89,11 @@ def test_set_pattern(completionview): model.set_pattern.assert_called_with('foo') +def test_set_pattern_no_model(completionview): + """Ensure that setting a pattern with no model does not fail.""" + completionview.set_pattern('foo') + + def test_maybe_update_geometry(completionview, config_stub, qtbot): """Ensure completion is resized only if shrink is True.""" with qtbot.assertNotEmitted(completionview.update_geometry): @@ -233,4 +238,4 @@ def test_completion_item_del_no_selection(completionview): completionview.set_model(model) with pytest.raises(cmdexc.CommandError, match='No item selected!'): completionview.completion_item_del() - func.assert_not_called() + assert not func.called From 1aed2470e5912264c8c90be58b97a729fd0e8bdf Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 12 Jul 2017 22:14:27 -0400 Subject: [PATCH 251/337] SQL code review. - Fix flake8 - history.clear should also clear completion table - call _resize_columns in set_model, not set_pattern - add more unit-testing for the history completion table --- .flake8 | 4 ++-- qutebrowser/browser/history.py | 1 + qutebrowser/completion/completionwidget.py | 3 ++- tests/unit/browser/test_history.py | 16 ++++++++++++++-- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.flake8 b/.flake8 index b87fef8b2..eada2c86d 100644 --- a/.flake8 +++ b/.flake8 @@ -28,7 +28,7 @@ exclude = .*,__pycache__,resources.py ignore = E128,E226,E265,E501,E402,E266,E722,E731, F401, - N802,N806 + N802, P101,P102,P103, D102,D103,D104,D105,D209,D211,D402,D403 min-version = 3.4.0 @@ -39,7 +39,7 @@ putty-ignore = /# pragma: no mccabe/ : +C901 tests/*/test_*.py : +D100,D101,D401 tests/conftest.py : +F403 - tests/unit/browser/webkit/test_history.py : +N806 + tests/unit/browser/test_history.py : +N806 tests/helpers/fixtures.py : +N806 tests/unit/browser/webkit/http/test_content_disposition.py : +D400 scripts/dev/ci/appveyor_install.py : +FI53 diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index b5b8af149..b9e791207 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -115,6 +115,7 @@ class WebHistory(sql.SqlTable): def _do_clear(self): self.delete_all() + self.completion.delete_all() def delete_url(self, url): """Remove all history entries with the given url. diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 68466630a..6e1e51680 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -286,16 +286,17 @@ class CompletionView(QTreeView): self._active = True self._maybe_show() + self._resize_columns() for i in range(model.rowCount()): self.expand(model.index(i, 0)) def set_pattern(self, pattern): + """Set the pattern on the underlying model.""" if not self.model(): return self.pattern = pattern with debug.log_time(log.completion, 'Set pattern {}'.format(pattern)): self.model().set_pattern(pattern) - self._resize_columns() self._maybe_update_geometry() self._maybe_show() diff --git a/tests/unit/browser/test_history.py b/tests/unit/browser/test_history.py index 5a8425646..81637f3d4 100644 --- a/tests/unit/browser/test_history.py +++ b/tests/unit/browser/test_history.py @@ -123,6 +123,7 @@ def test_clear_force(qtbot, tmpdir, hist): hist.add_url(QUrl('http://www.qutebrowser.org/')) hist.clear(force=True) assert not len(hist) + assert not len(hist.completion) def test_delete_url(hist): @@ -131,10 +132,16 @@ def test_delete_url(hist): hist.add_url(QUrl('http://example.com/2'), atime=0) before = set(hist) + completion_before = set(hist.completion) + hist.delete_url(QUrl('http://example.com/1')) + diff = before.difference(set(hist)) assert diff == {('http://example.com/1', '', 0, False)} + completion_diff = completion_before.difference(set(hist.completion)) + assert completion_diff == {('http://example.com/1', '', 0)} + @pytest.mark.parametrize('url, atime, title, redirect, expected_url', [ ('http://www.example.com', 12346, 'the title', False, @@ -146,15 +153,20 @@ def test_delete_url(hist): ('https://user:pass@example.com', 12346, 'the title', False, 'https://user@example.com'), ]) -def test_add_item(qtbot, hist, url, atime, title, redirect, expected_url): +def test_add_url(qtbot, hist, url, atime, title, redirect, expected_url): hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect) assert list(hist) == [(expected_url, title, atime, redirect)] + if redirect: + assert not len(hist.completion) + else: + assert list(hist.completion) == [(expected_url, title, atime)] -def test_add_item_invalid(qtbot, hist, caplog): +def test_add_url_invalid(qtbot, hist, caplog): with caplog.at_level(logging.WARNING): hist.add_url(QUrl()) assert not list(hist) + assert not list(hist.completion) @pytest.mark.parametrize('level, url, req_url, expected', [ From 7dfca608932ee3f55b5df4b5691458e1bbb81f04 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Wed, 12 Jul 2017 19:45:14 -0700 Subject: [PATCH 252/337] Refactor tab_close_prompt_if_pinned Now it lives in tabbedbrowser.py as method instead of a static function --- qutebrowser/browser/commands.py | 21 ++++----------------- qutebrowser/mainwindow/tabbedbrowser.py | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 229f93437..4c6f84ba0 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -227,20 +227,6 @@ class CommandDispatcher: self._tabbed_browser.close_tab(tab) tabbar.setSelectionBehaviorOnRemove(old_selection_behavior) - @staticmethod - def tab_close_prompt_if_pinned(tab, force, yes_action): - """Helper method for tab_close. - - If tab is pinned, prompt. If everything is good, run yes_action. - """ - if tab.data.pinned and not force: - message.confirm_async( - title='Pinned Tab', - text="Are you sure you want to close a pinned tab?", - yes_action=yes_action, default=False) - else: - yes_action() - @cmdutils.register(instance='command-dispatcher', scope='window') @cmdutils.argument('count', count=True) def tab_close(self, prev=False, next_=False, opposite=False, @@ -261,7 +247,7 @@ class CommandDispatcher: close = functools.partial(self._tab_close, tab, prev, next_, opposite) - CommandDispatcher.tab_close_prompt_if_pinned(tab, force, close) + self._tabbed_browser.tab_close_prompt_if_pinned(tab, force, close) @cmdutils.register(instance='command-dispatcher', scope='window', name='tab-pin') @@ -919,8 +905,9 @@ class CommandDispatcher: if not force: for i, tab in enumerate(self._tabbed_browser.widgets()): if _to_close(i) and tab.data.pinned: - CommandDispatcher.tab_close_prompt_if_pinned( - tab, force, + self._tabbed_browser.tab_close_prompt_if_pinned( + tab, + force, lambda: self.tab_only( prev=prev, next_=next_, force=True)) return diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index 600b2814d..5c0cd16f8 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -26,7 +26,6 @@ from PyQt5.QtWidgets import QSizePolicy from PyQt5.QtCore import pyqtSignal, pyqtSlot, QTimer, QUrl, QSize from PyQt5.QtGui import QIcon -from qutebrowser.browser.commands import CommandDispatcher from qutebrowser.config import config from qutebrowser.keyinput import modeman from qutebrowser.mainwindow import tabwidget @@ -233,6 +232,19 @@ class TabbedBrowser(tabwidget.TabWidget): for tab in self.widgets(): self._remove_tab(tab) + def tab_close_prompt_if_pinned(self, tab, force, yes_action): + """Helper method for tab_close. + + If tab is pinned, prompt. If everything is good, run yes_action. + """ + if tab.data.pinned and not force: + message.confirm_async( + title='Pinned Tab', + text="Are you sure you want to close a pinned tab?", + yes_action=yes_action, default=False) + else: + yes_action() + def close_tab(self, tab, *, add_undo=True): """Close a tab. @@ -367,7 +379,7 @@ class TabbedBrowser(tabwidget.TabWidget): log.webview.debug("Got invalid tab {} for index {}!".format( tab, idx)) return - CommandDispatcher.tab_close_prompt_if_pinned( + self.tab_close_prompt_if_pinned( tab, False, lambda: self.close_tab(tab)) @pyqtSlot(browsertab.AbstractTab) From 9898c1ba4bdacea90f6325320555cca5f340436b Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 13 Jul 2017 11:02:26 +0200 Subject: [PATCH 253/337] Update docs --- CHANGELOG.asciidoc | 1 + README.asciidoc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d3bc99b2a..97be72bda 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -65,6 +65,7 @@ Fixes - Fixed empty space being shown after tabs in the tabbar in some cases. - Fixed `:restart` in private browsing mode. - Fixed printing on macOS. +- Closing a pinned tab via mouse now also prompts for confirmation. v0.11.0 ------- diff --git a/README.asciidoc b/README.asciidoc index 530379242..8be36e2bf 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -162,8 +162,8 @@ Contributors, sorted by the number of commits in descending order: * Daniel Karbach * Kevin Velghe * Raphael Pierzina -* Joel Torstensson * Jay Kamat +* Joel Torstensson * Patric Schmitz * Fritz Reichwald * Tarcisio Fedrizzi From 8745f80d90892c354bf82dc2058015a3b0e5156f Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 13 Jul 2017 08:54:21 -0400 Subject: [PATCH 254/337] Fix qute://history SQL bug. The javascript history page was requesting the new start_time in ms, but the python code was expecting seconds. This is fixed by removing all the millisecond translations in the python code and only translating to milliseconds in the javascript code that formats dates. --- qutebrowser/browser/qutescheme.py | 6 ++++-- qutebrowser/javascript/history.js | 3 ++- tests/unit/browser/test_qutescheme.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 0e05cd7fb..0f86d0980 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -192,6 +192,8 @@ def history_data(start_time, offset=None): start_time: select history starting from this timestamp. offset: number of items to skip """ + # history atimes are stored as ints, ensure start_time is not a float + start_time = int(start_time) hist = objreg.get('web-history') if offset is not None: entries = hist.entries_before(start_time, limit=1000, offset=offset) @@ -200,7 +202,7 @@ def history_data(start_time, offset=None): end_time = start_time - 24*60*60 entries = hist.entries_between(end_time, start_time) - return [{"url": e.url, "title": e.title or e.url, "time": e.atime * 1000} + return [{"url": e.url, "title": e.title or e.url, "time": e.atime} for e in entries] @@ -252,7 +254,7 @@ def qute_history(url): start_time = time.mktime(next_date.timetuple()) - 1 history = [ (i["url"], i["title"], - datetime.datetime.fromtimestamp(i["time"]/1000), + datetime.datetime.fromtimestamp(i["time"]), QUrl(i["url"]).host()) for i in history_data(start_time) ] diff --git a/qutebrowser/javascript/history.js b/qutebrowser/javascript/history.js index 3f5f9def6..26b4405e9 100644 --- a/qutebrowser/javascript/history.js +++ b/qutebrowser/javascript/history.js @@ -174,7 +174,8 @@ window.loadHistory = (function() { for (var i = 0, len = history.length; i < len; i++) { var item = history[i]; - var currentItemDate = new Date(item.time); + // python's time.time returns seconds, but js Date expects ms + var currentItemDate = new Date(item.time * 1000); getSessionNode(currentItemDate).appendChild(makeHistoryRow( item.url, item.title, currentItemDate.toLocaleTimeString() )); diff --git a/tests/unit/browser/test_qutescheme.py b/tests/unit/browser/test_qutescheme.py index 154af0355..693b2607c 100644 --- a/tests/unit/browser/test_qutescheme.py +++ b/tests/unit/browser/test_qutescheme.py @@ -129,8 +129,8 @@ class TestHistoryHandler: # test times end_time = start_time - 24*60*60 for item in items: - assert item['time'] <= start_time * 1000 - assert item['time'] > end_time * 1000 + assert item['time'] <= start_time + assert item['time'] > end_time def test_qute_history_benchmark(self, fake_web_history, benchmark, now): r = range(100000) From 5c367e7ab24bb3b885afe0eb7c1c95ad0caf2634 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 13 Jul 2017 17:26:58 +0200 Subject: [PATCH 255/337] Fix the "try again" button on error pages Fixes #2810 --- CHANGELOG.asciidoc | 1 + qutebrowser/html/error.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 97be72bda..89a42747a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -66,6 +66,7 @@ Fixes - Fixed `:restart` in private browsing mode. - Fixed printing on macOS. - Closing a pinned tab via mouse now also prompts for confirmation. +- The "try again" button on error pages works correctly again. v0.11.0 ------- diff --git a/qutebrowser/html/error.html b/qutebrowser/html/error.html index 06261a06a..615e4ba8b 100644 --- a/qutebrowser/html/error.html +++ b/qutebrowser/html/error.html @@ -61,7 +61,7 @@ li { {{ super() }} function tryagain() { - location.href = url; + location.href = "{{ url }}"; } {% endblock %} From ee1707c4d4e85f0ee05da0912c57b398c692c2a0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 16 Jul 2017 20:20:33 +0200 Subject: [PATCH 256/337] Update back/forward indicator on tab switches --- qutebrowser/mainwindow/statusbar/backforward.py | 4 ++++ qutebrowser/mainwindow/statusbar/bar.py | 1 + 2 files changed, 5 insertions(+) diff --git a/qutebrowser/mainwindow/statusbar/backforward.py b/qutebrowser/mainwindow/statusbar/backforward.py index 302de4d95..fe044e621 100644 --- a/qutebrowser/mainwindow/statusbar/backforward.py +++ b/qutebrowser/mainwindow/statusbar/backforward.py @@ -34,6 +34,10 @@ class Backforward(textbase.TextBase): self.setText('') self.hide() return + self.on_tab_changed(tab) + + def on_tab_changed(self, tab): + """Update the text based on the given tab.""" text = '' if tab.history.can_go_back(): text += '<' diff --git a/qutebrowser/mainwindow/statusbar/bar.py b/qutebrowser/mainwindow/statusbar/bar.py index 64a084327..c6fafe14f 100644 --- a/qutebrowser/mainwindow/statusbar/bar.py +++ b/qutebrowser/mainwindow/statusbar/bar.py @@ -325,6 +325,7 @@ class StatusBar(QWidget): self.url.on_tab_changed(tab) self.prog.on_tab_changed(tab) self.percentage.on_tab_changed(tab) + self.backforward.on_tab_changed(tab) self.maybe_hide() assert tab.private == self._color_flags.private From c32d452786508c9bc7f27f1e8c660001b310eaa9 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 14 Jul 2017 09:28:06 -0400 Subject: [PATCH 257/337] Add LIMIT to history query. For performance, re-introduce web-history-max-items. As the history query has now become a very specific multi-part query and history completion was the only consumer of SqlCategory, SqlCategory is now replaced by a HistoryCategory class. --- .../{sqlcategory.py => histcategory.py} | 72 ++++---- qutebrowser/completion/models/urlmodel.py | 13 +- qutebrowser/config/config.py | 1 - qutebrowser/config/configdata.py | 5 + tests/unit/completion/test_histcategory.py | 150 +++++++++++++++++ tests/unit/completion/test_models.py | 28 ++-- tests/unit/completion/test_sqlcategory.py | 158 ------------------ 7 files changed, 204 insertions(+), 223 deletions(-) rename qutebrowser/completion/models/{sqlcategory.py => histcategory.py} (54%) create mode 100644 tests/unit/completion/test_histcategory.py delete mode 100644 tests/unit/completion/test_sqlcategory.py diff --git a/qutebrowser/completion/models/sqlcategory.py b/qutebrowser/completion/models/histcategory.py similarity index 54% rename from qutebrowser/completion/models/sqlcategory.py rename to qutebrowser/completion/models/histcategory.py index 2664edbfe..1899c2805 100644 --- a/qutebrowser/completion/models/sqlcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""A completion model backed by SQL tables.""" +"""A completion category that queries the SQL History store.""" import re @@ -26,50 +26,50 @@ from PyQt5.QtSql import QSqlQueryModel from qutebrowser.misc import sql from qutebrowser.utils import debug from qutebrowser.commands import cmdexc +from qutebrowser.config import config -class SqlCategory(QSqlQueryModel): +class HistoryCategory(QSqlQueryModel): - """Wraps a SqlQuery for use as a completion category.""" + """A completion category that queries the SQL History store.""" - def __init__(self, name, *, title=None, filter_fields, sort_by=None, - sort_order=None, select='*', - delete_func=None, parent=None): - """Create a new completion category backed by a sql table. - - Args: - name: Name of the table in the database. - title: Title of category, defaults to table name. - filter_fields: Names of fields to apply filter pattern to. - select: A custom result column expression for the select statement. - sort_by: The name of the field to sort by, or None for no sorting. - sort_order: Either 'asc' or 'desc', if sort_by is non-None - delete_func: Callback to delete a selected item. - """ + def __init__(self, *, delete_func=None, parent=None): + """Create a new History completion category.""" super().__init__(parent=parent) - self.name = title or name + self.name = "History" - querystr = 'select {} from {} where ('.format(select, name) - # the incoming pattern will have literal % and _ escaped with '\' - # we need to tell sql to treat '\' as an escape character - querystr += ' or '.join("{} like :pattern escape '\\'".format(f) - for f in filter_fields) - querystr += ')' + # replace ' to avoid breaking the query + timefmt = "strftime('{}', last_atime, 'unixepoch')".format( + config.get('completion', 'timestamp-format').replace("'", "`")) - if sort_by: - assert sort_order in ['asc', 'desc'], sort_order - querystr += ' order by {} {}'.format(sort_by, sort_order) - else: - assert sort_order is None, sort_order + self._query = sql.Query(' '.join([ + "SELECT url, title, {}".format(timefmt), + "FROM CompletionHistory", + # the incoming pattern will have literal % and _ escaped with '\' + # we need to tell sql to treat '\' as an escape character + "WHERE (url LIKE :pat escape '\\' or title LIKE :pat escape '\\')", + self._atime_expr(), + "ORDER BY last_atime DESC", + ]), forward_only=False) - self._query = sql.Query(querystr, forward_only=False) - - # map filter_fields to indices - col_query = sql.Query('SELECT * FROM {} LIMIT 1'.format(name)) - rec = col_query.run().record() - self.columns_to_filter = [rec.indexOf(n) for n in filter_fields] + # advertise that this model filters by URL and title + self.columns_to_filter = [0, 1] self.delete_func = delete_func + def _atime_expr(self): + """If max_items is set, return an expression to limit the query.""" + max_items = config.get('completion', 'web-history-max-items') + if max_items < 0: + return '' + + min_atime = sql.Query(' '.join([ + 'SELECT min(last_atime) FROM', + '(SELECT last_atime FROM CompletionHistory', + 'ORDER BY last_atime DESC LIMIT :limit)', + ])).run(limit=max_items).value() + + return "AND last_atime >= {}".format(min_atime) + def set_pattern(self, pattern): """Set the pattern used to filter results. @@ -83,7 +83,7 @@ class SqlCategory(QSqlQueryModel): pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) with debug.log_time('sql', 'Running completion query'): - self._query.run(pattern=pattern) + self._query.run(pat=pattern) self.setQuery(self._query) def delete_cur_item(self, index): diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 4f5fdeae9..0c5fbeacc 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -20,8 +20,7 @@ """Function to return the url completion model for the `open` command.""" from qutebrowser.completion.models import (completionmodel, listcategory, - sqlcategory) -from qutebrowser.config import config + histcategory) from qutebrowser.utils import log, objreg @@ -66,14 +65,6 @@ def url(): model.add_category(listcategory.ListCategory( 'Bookmarks', bookmarks, delete_func=_delete_bookmark)) - # replace 's to avoid breaking the query - timefmt = config.get('completion', 'timestamp-format').replace("'", "`") - select_time = "strftime('{}', last_atime, 'unixepoch')".format(timefmt) - hist_cat = sqlcategory.SqlCategory( - 'CompletionHistory', title='History', - sort_order='desc', sort_by='last_atime', - filter_fields=['url', 'title'], - select='url, title, {}'.format(select_time), - delete_func=_delete_history) + hist_cat = histcategory.HistoryCategory(delete_func=_delete_history) model.add_category(hist_cat) return model diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index 1245429c2..8bae2bae0 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -382,7 +382,6 @@ class ConfigManager(QObject): ('storage', 'offline-storage-default-quota'), ('storage', 'offline-web-application-cache-quota'), ('content', 'css-regions'), - ('completion', 'web-history-max-items'), ] CHANGED_OPTIONS = { ('content', 'cookies-accept'): diff --git a/qutebrowser/config/configdata.py b/qutebrowser/config/configdata.py index 6af04d9f4..23d3efb67 100644 --- a/qutebrowser/config/configdata.py +++ b/qutebrowser/config/configdata.py @@ -502,6 +502,11 @@ def data(readonly=False): "How many commands to save in the command history.\n\n" "0: no history / -1: unlimited"), + ('web-history-max-items', + SettingValue(typ.Int(minval=-1), '-1'), + "How many URLs to show in the web history.\n\n" + "0: no history / -1: unlimited"), + ('quick-complete', SettingValue(typ.Bool(), 'true'), "Whether to move on to the next part when there's only one " diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py new file mode 100644 index 000000000..f285c1dcc --- /dev/null +++ b/tests/unit/completion/test_histcategory.py @@ -0,0 +1,150 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Test the web history completion category.""" + +import unittest.mock +import time + +import pytest + +from qutebrowser.misc import sql +from qutebrowser.completion.models import histcategory +from qutebrowser.commands import cmdexc + + +@pytest.fixture +def hist(init_sql, config_stub): + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', + 'web-history-max-items': -1} + return sql.SqlTable('CompletionHistory', ['url', 'title', 'last_atime']) + + +@pytest.mark.parametrize('pattern, before, after', [ + ('foo', + [('foo', ''), ('bar', ''), ('aafobbb', '')], + [('foo',)]), + + ('FOO', + [('foo', ''), ('bar', ''), ('aafobbb', '')], + [('foo',)]), + + ('foo', + [('FOO', ''), ('BAR', ''), ('AAFOBBB', '')], + [('FOO',)]), + + ('foo', + [('baz', 'bar'), ('foo', ''), ('bar', 'foo')], + [('foo', ''), ('bar', 'foo')]), + + ('foo', + [('fooa', ''), ('foob', ''), ('fooc', '')], + [('fooa', ''), ('foob', ''), ('fooc', '')]), + + ('foo', + [('foo', 'bar'), ('bar', 'foo'), ('biz', 'baz')], + [('foo', 'bar'), ('bar', 'foo')]), + + ('foo bar', + [('foo', ''), ('bar foo', ''), ('xfooyybarz', '')], + [('xfooyybarz', '')]), + + ('foo%bar', + [('foo%bar', ''), ('foo bar', ''), ('foobar', '')], + [('foo%bar', '')]), + + ('_', + [('a_b', ''), ('__a', ''), ('abc', '')], + [('a_b', ''), ('__a', '')]), + + ('%', + [('\\foo', '\\bar')], + []), + + ("can't", + [("can't touch this", ''), ('a', '')], + [("can't touch this", '')]), +]) +def test_set_pattern(pattern, before, after, model_validator, hist): + """Validate the filtering and sorting results of set_pattern.""" + for row in before: + hist.insert({'url': row[0], 'title': row[1], 'last_atime': 1}) + cat = histcategory.HistoryCategory() + model_validator.set_model(cat) + cat.set_pattern(pattern) + model_validator.validate(after) + + +@pytest.mark.parametrize('max_items, before, after', [ + (-1, [ + ('a', 'a', '2017-04-16'), + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ], [ + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ('a', 'a', '2017-04-16'), + ]), + (3, [ + ('a', 'a', '2017-04-16'), + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ], [ + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ('a', 'a', '2017-04-16'), + ]), + (2, [ + ('a', 'a', '2017-04-16'), + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ], [ + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ]) +]) +def test_sorting(max_items, before, after, model_validator, hist, config_stub): + """Validate the filtering and sorting results of set_pattern.""" + config_stub.data['completion']['web-history-max-items'] = max_items + for url, title, atime in before: + timestamp = time.mktime(time.strptime(atime, '%Y-%m-%d')) + hist.insert({'url': url, 'title': title, 'last_atime': timestamp}) + cat = histcategory.HistoryCategory() + model_validator.set_model(cat) + cat.set_pattern('') + model_validator.validate(after) + + +def test_delete_cur_item(hist): + hist.insert({'url': 'foo', 'title': 'Foo'}) + hist.insert({'url': 'bar', 'title': 'Bar'}) + func = unittest.mock.Mock(spec=[]) + cat = histcategory.HistoryCategory(delete_func=func) + cat.set_pattern('') + cat.delete_cur_item(cat.index(0, 0)) + func.assert_called_with(['foo', 'Foo', '']) + + +def test_delete_cur_item_no_func(hist): + hist.insert({'url': 'foo', 'title': 1}) + hist.insert({'url': 'bar', 'title': 2}) + cat = histcategory.HistoryCategory() + cat.set_pattern('') + with pytest.raises(cmdexc.CommandError, match='Cannot delete this item'): + cat.delete_cur_item(cat.index(0, 0)) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 4fab0c402..5b632eb3a 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -137,8 +137,10 @@ def bookmarks(bookmark_manager_stub): @pytest.fixture -def web_history(init_sql, stubs): +def web_history(init_sql, stubs, config_stub): """Fixture which provides a web-history object.""" + config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', + 'web-history-max-items': -1} stub = history.WebHistory() objreg.register('web-history', stub) yield stub @@ -266,7 +268,7 @@ def test_bookmark_completion(qtmodeltester, bookmarks): }) -def test_url_completion(qtmodeltester, config_stub, web_history_populated, +def test_url_completion(qtmodeltester, web_history_populated, quickmarks, bookmarks): """Test the results of url completion. @@ -275,7 +277,6 @@ def test_url_completion(qtmodeltester, config_stub, web_history_populated, - entries are sorted by access time - only the most recent entry is included for each url """ - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() model.set_pattern('') qtmodeltester.data_display_may_return_none = True @@ -318,11 +319,10 @@ def test_url_completion(qtmodeltester, config_stub, web_history_populated, ('foo%bar', '', '%', 1), ('foobar', '', '%', 0), ]) -def test_url_completion_pattern(config_stub, web_history, - quickmark_manager_stub, bookmark_manager_stub, - url, title, pattern, rowcount): +def test_url_completion_pattern(web_history, quickmark_manager_stub, + bookmark_manager_stub, url, title, pattern, + rowcount): """Test that url completion filters by url and title.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} web_history.add_url(QUrl(url), title) model = urlmodel.url() model.set_pattern(pattern) @@ -330,10 +330,9 @@ def test_url_completion_pattern(config_stub, web_history, assert model.rowCount(model.index(2, 0)) == rowcount -def test_url_completion_delete_bookmark(qtmodeltester, config_stub, bookmarks, +def test_url_completion_delete_bookmark(qtmodeltester, bookmarks, web_history, quickmarks): """Test deleting a bookmark from the url completion model.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() model.set_pattern('') qtmodeltester.data_display_may_return_none = True @@ -353,11 +352,10 @@ def test_url_completion_delete_bookmark(qtmodeltester, config_stub, bookmarks, assert len_before == len(bookmarks.marks) + 1 -def test_url_completion_delete_quickmark(qtmodeltester, config_stub, +def test_url_completion_delete_quickmark(qtmodeltester, quickmarks, web_history, bookmarks, qtbot): """Test deleting a bookmark from the url completion model.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() model.set_pattern('') qtmodeltester.data_display_may_return_none = True @@ -377,11 +375,10 @@ def test_url_completion_delete_quickmark(qtmodeltester, config_stub, assert len_before == len(quickmarks.marks) + 1 -def test_url_completion_delete_history(qtmodeltester, config_stub, +def test_url_completion_delete_history(qtmodeltester, web_history_populated, quickmarks, bookmarks): """Test deleting a history entry.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d'} model = urlmodel.url() model.set_pattern('') qtmodeltester.data_display_may_return_none = True @@ -598,14 +595,11 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, }) -def test_url_completion_benchmark(benchmark, config_stub, +def test_url_completion_benchmark(benchmark, quickmark_manager_stub, bookmark_manager_stub, web_history): """Benchmark url completion.""" - config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d', - 'web-history-max-items': 1000} - r = range(100000) entries = { 'last_atime': list(r), diff --git a/tests/unit/completion/test_sqlcategory.py b/tests/unit/completion/test_sqlcategory.py deleted file mode 100644 index a9321e33e..000000000 --- a/tests/unit/completion/test_sqlcategory.py +++ /dev/null @@ -1,158 +0,0 @@ -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) -# -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -"""Test SQL-based completions.""" - -import unittest.mock - -import pytest - -from qutebrowser.misc import sql -from qutebrowser.completion.models import sqlcategory -from qutebrowser.commands import cmdexc - - -pytestmark = pytest.mark.usefixtures('init_sql') - - -@pytest.mark.parametrize('sort_by, sort_order, data, expected', [ - (None, None, - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')]), - - ('a', 'asc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - - ('a', 'desc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - - ('b', 'asc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('C', 'A', 'G'), ('B', 'C', 'D'), ('A', 'F', 'C')]), - - ('b', 'desc', - [('B', 'C', 'D'), ('A', 'F', 'C'), ('C', 'A', 'G')], - [('A', 'F', 'C'), ('B', 'C', 'D'), ('C', 'A', 'G')]), - - ('c', 'asc', - [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], - [('A', 'F', 0), ('C', 'A', 1), ('B', 'C', 2)]), - - ('c', 'desc', - [('B', 'C', 2), ('A', 'F', 0), ('C', 'A', 1)], - [('B', 'C', 2), ('C', 'A', 1), ('A', 'F', 0)]), -]) -def test_sorting(sort_by, sort_order, data, expected, model_validator): - table = sql.SqlTable('Foo', ['a', 'b', 'c']) - for row in data: - table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], sort_by=sort_by, - sort_order=sort_order) - model_validator.set_model(cat) - cat.set_pattern('') - model_validator.validate(expected) - - -@pytest.mark.parametrize('pattern, filter_cols, before, after', [ - ('foo', [0], - [('foo', '', ''), ('bar', '', ''), ('aafobbb', '', '')], - [('foo',)]), - - ('foo', [0], - [('baz', 'bar', 'foo'), ('foo', '', ''), ('bar', 'foo', '')], - [('foo', '', '')]), - - ('foo', [0], - [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')], - [('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]), - - ('foo', [1], - [('foo', 'bar', ''), ('bar', 'foo', '')], - [('bar', 'foo', '')]), - - ('foo', [0, 1], - [('foo', 'bar', ''), ('bar', 'foo', ''), ('biz', 'baz', 'foo')], - [('foo', 'bar', ''), ('bar', 'foo', '')]), - - ('foo', [0, 1, 2], - [('foo', '', ''), ('bar', '', ''), ('baz', 'bar', 'foo')], - [('foo', '', ''), ('baz', 'bar', 'foo')]), - - ('foo bar', [0], - [('foo', '', ''), ('bar foo', '', ''), ('xfooyybarz', '', '')], - [('xfooyybarz', '', '')]), - - ('foo%bar', [0], - [('foo%bar', '', ''), ('foo bar', '', ''), ('foobar', '', '')], - [('foo%bar', '', '')]), - - ('_', [0], - [('a_b', '', ''), ('__a', '', ''), ('abc', '', '')], - [('a_b', '', ''), ('__a', '', '')]), - - ('%', [0, 1], - [('\\foo', '\\bar', '')], - []), - - ("can't", [0], - [("can't touch this", '', ''), ('a', '', '')], - [("can't touch this", '', '')]), -]) -def test_set_pattern(pattern, filter_cols, before, after, model_validator): - """Validate the filtering and sorting results of set_pattern.""" - table = sql.SqlTable('Foo', ['a', 'b', 'c']) - for row in before: - table.insert({'a': row[0], 'b': row[1], 'c': row[2]}) - filter_fields = [['a', 'b', 'c'][i] for i in filter_cols] - cat = sqlcategory.SqlCategory('Foo', filter_fields=filter_fields) - model_validator.set_model(cat) - cat.set_pattern(pattern) - model_validator.validate(after) - - -def test_select(model_validator): - table = sql.SqlTable('Foo', ['a', 'b', 'c']) - table.insert({'a': 'foo', 'b': 'bar', 'c': 'baz'}) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], select='b, c, a') - model_validator.set_model(cat) - cat.set_pattern('') - model_validator.validate([('bar', 'baz', 'foo')]) - - -def test_delete_cur_item(): - table = sql.SqlTable('Foo', ['a', 'b']) - table.insert({'a': 'foo', 'b': 1}) - table.insert({'a': 'bar', 'b': 2}) - func = unittest.mock.Mock(spec=[]) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a'], delete_func=func) - cat.set_pattern('') - cat.delete_cur_item(cat.index(0, 0)) - func.assert_called_with(['foo', 1]) - - -def test_delete_cur_item_no_func(): - table = sql.SqlTable('Foo', ['a', 'b']) - table.insert({'a': 'foo', 'b': 1}) - table.insert({'a': 'bar', 'b': 2}) - cat = sqlcategory.SqlCategory('Foo', filter_fields=['a']) - cat.set_pattern('') - with pytest.raises(cmdexc.CommandError, match='Cannot delete this item'): - cat.delete_cur_item(cat.index(0, 0)) From f45acaa9c8104a865b2d5790f6e26ff34cbee77a Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 17 Jul 2017 08:37:24 -0400 Subject: [PATCH 258/337] Fix coverage check for sqlcategory rename. --- scripts/dev/check_coverage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index 950f21ff1..e63bb821f 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -156,8 +156,8 @@ PERFECT_FILES = [ ('tests/unit/completion/test_models.py', 'completion/models/urlmodel.py'), - ('tests/unit/completion/test_sqlcategory.py', - 'completion/models/sqlcategory.py'), + ('tests/unit/completion/test_histcategory.py', + 'completion/models/histcategory.py'), ] From 4a7fe25f6623126986782b5ee1bc61f7a6497354 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 17 Jul 2017 14:58:02 +0200 Subject: [PATCH 259/337] Only clear search with :search if one is displayed For some reason, calling search.clear() while no search is displayed causes the backends to un-focus inputs, and with QtWebKit, even hinting can't focus them again after that. --- CHANGELOG.asciidoc | 2 ++ qutebrowser/browser/commands.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 89a42747a..5e0ad3ee8 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -49,6 +49,8 @@ Changed - Using `:download` now uses the page's title as filename. - Using `:back` or `:forward` with a count now skips intermediate pages. - When there are multiple messages shown, the timeout is increased. +- `:search` now only clears the search if one was displayed before, so pressing + `` doesn't un-focus inputs anymore. Fixes ~~~~~ diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 8b6a688c1..b3e9c37bf 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1744,7 +1744,8 @@ class CommandDispatcher: """ self.set_mark("'") tab = self._current_widget() - tab.search.clear() + if tab.search.search_displayed: + tab.search.clear() if not text: return From bdfea0fa6ff12d12256d5c49ac40a9da7f04b23d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 17 Jul 2017 16:02:14 +0200 Subject: [PATCH 260/337] Update setuptools from 36.0.1 to 36.2.0 --- misc/requirements/requirements-pip.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pip.txt b/misc/requirements/requirements-pip.txt index 9bae64d6d..55c1f873e 100644 --- a/misc/requirements/requirements-pip.txt +++ b/misc/requirements/requirements-pip.txt @@ -3,6 +3,6 @@ appdirs==1.4.3 packaging==16.8 pyparsing==2.2.0 -setuptools==36.0.1 +setuptools==36.2.0 six==1.10.0 wheel==0.29.0 From db2f60b0efd2fa16f9d5e82234965b2870a1db7a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 17 Jul 2017 16:02:16 +0200 Subject: [PATCH 261/337] Update decorator from 4.0.11 to 4.1.1 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 40de7a673..f9d734e7a 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -5,7 +5,7 @@ cheroot==5.7.0 click==6.7 # colorama==0.3.9 coverage==4.4.1 -decorator==4.0.11 +decorator==4.1.1 EasyProcess==0.2.3 fields==5.0.0 Flask==0.12.2 From 72de0fcfcb1854b85145e449c159aa1909eb2e1e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 17 Jul 2017 16:02:17 +0200 Subject: [PATCH 262/337] Update hypothesis from 3.12.0 to 3.13.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index f9d734e7a..a36807b69 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -12,7 +12,7 @@ Flask==0.12.2 glob2==0.5 httpbin==0.5.0 hunter==1.4.1 -hypothesis==3.12.0 +hypothesis==3.13.0 itsdangerous==0.24 # Jinja2==2.9.6 Mako==1.0.6 From a976e9011dba91e0fba2b8a683674d9de4899a59 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 17 Jul 2017 16:02:18 +0200 Subject: [PATCH 263/337] Update mako from 1.0.6 to 1.0.7 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index a36807b69..746ce9b89 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -15,7 +15,7 @@ hunter==1.4.1 hypothesis==3.13.0 itsdangerous==0.24 # Jinja2==2.9.6 -Mako==1.0.6 +Mako==1.0.7 # MarkupSafe==1.0 parse==1.8.2 parse-type==0.3.4 From d9f0e21ea429219977d1ffb7264e1980f1f32408 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 17 Jul 2017 16:02:20 +0200 Subject: [PATCH 264/337] Update pytest-qt from 2.1.0 to 2.1.2 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 746ce9b89..207b71a67 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -28,7 +28,7 @@ pytest-cov==2.5.1 pytest-faulthandler==1.3.1 pytest-instafail==0.3.0 pytest-mock==1.6.0 -pytest-qt==2.1.0 +pytest-qt==2.1.2 pytest-repeat==0.4.1 pytest-rerunfailures==2.2 pytest-travis-fold==1.2.0 From 19d8411c15181e6482b7a791b6909943766a8159 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 17 Jul 2017 16:02:21 +0200 Subject: [PATCH 265/337] Update vulture from 0.15 to 0.16 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 207b71a67..91cecb798 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -35,5 +35,5 @@ pytest-travis-fold==1.2.0 pytest-xvfb==1.0.0 PyVirtualDisplay==0.2.1 six==1.10.0 -vulture==0.15 +vulture==0.16 Werkzeug==0.12.2 From 4d356e5320a60ab32388e7a1fb4528e8b1e2f62f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 17 Jul 2017 16:02:22 +0200 Subject: [PATCH 266/337] Update vulture from 0.15 to 0.16 --- misc/requirements/requirements-vulture.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-vulture.txt b/misc/requirements/requirements-vulture.txt index 9edb87f9c..f2ba7dcbf 100644 --- a/misc/requirements/requirements-vulture.txt +++ b/misc/requirements/requirements-vulture.txt @@ -1,3 +1,3 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -vulture==0.15 +vulture==0.16 From a26fc89f4918968ec99d100ffd28a78e20393e42 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 19 Jul 2017 11:59:44 +0200 Subject: [PATCH 267/337] Simplify setting the size for background tabs We can simply look at the size of the existing open tab. --- qutebrowser/mainwindow/tabbedbrowser.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index 5c0cd16f8..df0048516 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -449,13 +449,7 @@ class TabbedBrowser(tabwidget.TabWidget): # Make sure the background tab has the correct initial size. # With a foreground tab, it's going to be resized correctly by the # layout anyways. - if self.tabBar().vertical: - tab_size = QSize(self.width() - self.tabBar().width(), - self.height()) - else: - tab_size = QSize(self.width(), - self.height() - self.tabBar().height()) - tab.resize(tab_size) + tab.resize(self.currentWidget().size()) self.tab_index_changed.emit(self.currentIndex(), self.count()) else: self.setCurrentWidget(tab) From a3834d043b85f71134fd9098829160223bb3d13b Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 19 Jul 2017 12:22:30 +0200 Subject: [PATCH 268/337] pytest: Set testpaths --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index ad1a56086..225d64d9a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,6 @@ [pytest] addopts = --strict -rfEw --faulthandler-timeout=70 --instafail --pythonwarnings error +testpaths = tests markers = gui: Tests using the GUI (e.g. spawning widgets) posix: Tests which only can run on a POSIX OS. From fafa063bcdaff2bf5bd39dab9c4bfa06599ddfc6 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 19 Jul 2017 12:55:51 +0200 Subject: [PATCH 269/337] Remove unused import --- qutebrowser/mainwindow/tabbedbrowser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index df0048516..d74f74b4a 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -23,7 +23,7 @@ import functools import collections from PyQt5.QtWidgets import QSizePolicy -from PyQt5.QtCore import pyqtSignal, pyqtSlot, QTimer, QUrl, QSize +from PyQt5.QtCore import pyqtSignal, pyqtSlot, QTimer, QUrl from PyQt5.QtGui import QIcon from qutebrowser.config import config From 8dbb61e9e3401a805d60a5c8aec3351082cb2492 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Wed, 19 Jul 2017 21:37:48 -0700 Subject: [PATCH 270/337] Enforce a minimum size for non-pinned tabs Closes #2826 --- qutebrowser/mainwindow/tabwidget.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 86fefaeb3..4756d231e 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -517,7 +517,9 @@ class TabBar(QTabBar): no_pinned_count = self.count() - self.pinned_count pinned_width = tab_width_pinned_conf * self.pinned_count - no_pinned_width = self.width() - pinned_width + # Prevent any tabs from being smaller than the min size + no_pinned_width = max(self.width() - pinned_width, + minimum_size.width() * no_pinned_count) if pinned: width = tab_width_pinned_conf From 5939bc990af47004ce372618e9f2b097a2d1cdfc Mon Sep 17 00:00:00 2001 From: Noor Christensen Date: Thu, 20 Jul 2017 14:36:27 +0200 Subject: [PATCH 271/337] Clarify dependecies for readability userscript --- misc/userscripts/readability | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/misc/userscripts/readability b/misc/userscripts/readability index 2de4be5ab..65b160695 100755 --- a/misc/userscripts/readability +++ b/misc/userscripts/readability @@ -2,6 +2,11 @@ # # Executes python-readability on current page and opens the summary as new tab. # +# Depends on the python-readability package, or its fork: +# +# - https://github.com/buriy/python-readability +# - https://github.com/bookieio/breadability +# # Usage: # :spawn --userscript readability # From 0eb347186cf0174166d6452a2420d20b511cecff Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 20 Jul 2017 08:59:12 -0400 Subject: [PATCH 272/337] Add 'localtime' to sql history query. We need to tell sqlite to convert the timestamps to localtime during formatting, otherwise it formats them as though you are in UTC. Also fix up a few uses of mktime. --- qutebrowser/browser/qutescheme.py | 2 +- qutebrowser/completion/models/histcategory.py | 7 ++++--- tests/unit/completion/test_histcategory.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 0f86d0980..e712adcf8 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -251,7 +251,7 @@ def qute_history(url): prev_date = curr_date - one_day # start_time is the last second of curr_date - start_time = time.mktime(next_date.timetuple()) - 1 + start_time = next_date.timestamp() - 1 history = [ (i["url"], i["title"], datetime.datetime.fromtimestamp(i["time"]), diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index 1899c2805..fa8443a60 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -38,9 +38,10 @@ class HistoryCategory(QSqlQueryModel): super().__init__(parent=parent) self.name = "History" - # replace ' to avoid breaking the query - timefmt = "strftime('{}', last_atime, 'unixepoch')".format( - config.get('completion', 'timestamp-format').replace("'", "`")) + # replace ' in timestamp-format to avoid breaking the query + timefmt = ("strftime('{}', last_atime, 'unixepoch', 'localtime')" + .format(config.get('completion', 'timestamp-format') + .replace("'", "`"))) self._query = sql.Query(' '.join([ "SELECT url, title, {}".format(timefmt), diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index f285c1dcc..0b5fcb915 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -20,7 +20,7 @@ """Test the web history completion category.""" import unittest.mock -import time +import datetime import pytest @@ -123,7 +123,7 @@ def test_sorting(max_items, before, after, model_validator, hist, config_stub): """Validate the filtering and sorting results of set_pattern.""" config_stub.data['completion']['web-history-max-items'] = max_items for url, title, atime in before: - timestamp = time.mktime(time.strptime(atime, '%Y-%m-%d')) + timestamp = datetime.datetime.strptime(atime, '%Y-%m-%d').timestamp() hist.insert({'url': url, 'title': title, 'last_atime': timestamp}) cat = histcategory.HistoryCategory() model_validator.set_model(cat) From a4e644c2857b015c53e1b459f95d9782f94fea12 Mon Sep 17 00:00:00 2001 From: Noor Christensen Date: Thu, 20 Jul 2017 16:21:47 +0200 Subject: [PATCH 273/337] Add support for breadability module in readability userscript --- misc/userscripts/readability | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/misc/userscripts/readability b/misc/userscripts/readability index 65b160695..639e3a111 100755 --- a/misc/userscripts/readability +++ b/misc/userscripts/readability @@ -12,15 +12,22 @@ # from __future__ import absolute_import import codecs, os -from readability.readability import Document tmpfile=os.path.expanduser('~/.local/share/qutebrowser/userscripts/readability.html') if not os.path.exists(os.path.dirname(tmpfile)): os.makedirs(os.path.dirname(tmpfile)) with codecs.open(os.environ['QUTE_HTML'], 'r', 'utf-8') as source: - doc = Document(source.read()) - content = doc.summary().replace('', '%s' % doc.title()) + data = source.read() + + try: + from breadability.readable import Article as reader + doc = reader(data) + content = doc.readable + except ImportError: + from readability import Document + doc = Document(data) + content = doc.summary().replace('', '%s' % doc.title()) with codecs.open(tmpfile, 'w', 'utf-8') as target: target.write('') From 1175543ce16c4f9e4925148307ca5a87ac69983e Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 20 Jul 2017 22:07:37 -0400 Subject: [PATCH 274/337] Fix qutescheme timestamp error. A date object doesn't have a timestamp property. Go back to using mktime. --- qutebrowser/browser/qutescheme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index e712adcf8..0f86d0980 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -251,7 +251,7 @@ def qute_history(url): prev_date = curr_date - one_day # start_time is the last second of curr_date - start_time = next_date.timestamp() - 1 + start_time = time.mktime(next_date.timetuple()) - 1 history = [ (i["url"], i["title"], datetime.datetime.fromtimestamp(i["time"]), From 7e36310e8a843a72690adbb88c79b209463f37e8 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 21 Jul 2017 13:18:20 +0200 Subject: [PATCH 275/337] Shorten and update README --- README.asciidoc | 278 ++++++++--------------------------- doc/help/commands.asciidoc | 2 +- doc/help/settings.asciidoc | 2 +- scripts/asciidoc2html.py | 2 - scripts/dev/ci/travis_run.sh | 1 - scripts/dev/src2asciidoc.py | 38 +---- 6 files changed, 61 insertions(+), 262 deletions(-) diff --git a/README.asciidoc b/README.asciidoc index 8be36e2bf..1604b85ec 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -36,11 +36,8 @@ Downloads --------- See the https://github.com/qutebrowser/qutebrowser/releases[github releases -page] for available downloads (currently a source archive, and standalone -packages as well as MSI installers for Windows). - -See link:INSTALL.asciidoc[INSTALL] for detailed instructions on how to get -qutebrowser running for various platforms. +page] for available downloads and the link:INSTALL.asciidoc[INSTALL] file for +detailed instructions on how to get qutebrowser running on various platforms. Documentation ------------- @@ -98,11 +95,16 @@ Requirements The following software and libraries are required to run qutebrowser: -* http://www.python.org/[Python] 3.4 or newer (3.5 recommended) -* http://qt.io/[Qt] 5.2.0 or newer (5.9 recommended) -* QtWebKit (old or link:https://github.com/annulen/webkit/wiki[reloaded]/NG) or QtWebEngine +* http://www.python.org/[Python] 3.4 or newer (3.6 recommended) - note that + Python 3.4 support will be dropped soon. +* http://qt.io/[Qt] 5.2.0 or newer (5.9 recommended) - note that support for Qt + < 5.7.1 will be dropped soon. +* QtWebKit (old or link:https://github.com/annulen/webkit/wiki[reloaded]/NG) or + QtWebEngine. Note that support for legacy QtWebKit (before 5.212) will be + dropped soon. * http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.2.0 or newer -(5.9 recommended) for Python 3 +(5.9 recommended) for Python 3. Note that support for PyQt < 5.7 will be dropped +soon. * https://pypi.python.org/pypi/setuptools/[pkg_resources/setuptools] * http://fdik.org/pyPEG/[pyPEG2] * http://jinja.pocoo.org/[jinja2] @@ -110,15 +112,14 @@ The following software and libraries are required to run qutebrowser: * http://pyyaml.org/wiki/PyYAML[PyYAML] * http://pyopengl.sourceforge.net/[PyOpenGL] when using QtWebEngine -The following libraries are optional and provide a better user experience: +The following libraries are optional: -* http://cthedot.de/cssutils/[cssutils] - -To generate the documentation for the `:help` command, when using the git -repository (rather than a release), http://asciidoc.org/[asciidoc] is needed. - -On Windows, https://pypi.python.org/pypi/colorama/[colorama] is needed to -display colored log output. +* http://cthedot.de/cssutils/[cssutils] (for an improved `:download --mhtml` + with QtWebKit) +* On Windows, https://pypi.python.org/pypi/colorama/[colorama] for colored log + output. +* http://asciidoc.org/[asciidoc] to generate the documentation for the `:help` + command, when using the git repository (rather than a release). See link:INSTALL.asciidoc[INSTALL] for directions on how to install qutebrowser and its dependencies. @@ -142,222 +143,59 @@ get in touch! Authors ------- -Contributors, sorted by the number of commits in descending order: +qutebrowser's primary author is Florian Bruhin (The Compiler), but qutebrowser +wouldn't be what it is without the help of +https://github.com/qutebrowser/qutebrowser/graphs/contributors[hundreds of contributors]! -// QUTE_AUTHORS_START -* Florian Bruhin -* Daniel Schadt -* Ryan Roden-Corrent -* Jan Verbeek -* Jakub Klinkovský -* Antoni Boucher -* Lamar Pavel -* Marshall Lochbaum -* Bruno Oliveira -* thuck -* Martin Tournoij -* Imran Sobir -* Alexander Cogneau -* Felix Van der Jeugt -* Daniel Karbach -* Kevin Velghe -* Raphael Pierzina -* Jay Kamat -* Joel Torstensson -* Patric Schmitz -* Fritz Reichwald -* Tarcisio Fedrizzi -* Claude -* Philipp Hansch -* Corentin Julé -* meles5 -* Panagiotis Ktistakis -* Artur Shaik -* Nathan Isom -* Thorsten Wißmann -* Austin Anderson -* Jimmy -* Iordanis Grigoriou -* Niklas Haas -* Maciej Wołczyk -* Clayton Craft -* sandrosc -* Alexey "Averrin" Nabrodov -* pkill9 -* nanjekyejoannah -* avk -* ZDarian -* Milan Svoboda -* John ShaggyTwoDope Jenkins -* Peter Vilim -* Jacob Sword -* knaggita -* Yashar Shahi -* Oliver Caldwell -* Nikolay Amiantov -* Marius -* Julian Weigt -* Tomasz Kramkowski -* Sebastian Frysztak -* Julie Engel -* Jonas Schürmann -* error800 -* Michael Hoang -* Liam BEGUIN -* Daniel Fiser -* skinnay -* Zach-Button -* Samuel Walladge -* Peter Rice -* Ismail S -* Halfwit -* David Vogt -* Claire Cavanaugh -* Christian Helbling -* rikn00 -* kanikaa1234 -* haitaka -* Nick Ginther -* Michał Góral -* Michael Ilsaas -* Martin Zimmermann -* Link -* Jussi Timperi -* Cosmin Popescu -* Brian Jackson -* sbinix -* rsteube -* neeasade -* jnphilipp -* Yannis Rohloff -* Tobias Patzl -* Stefan Tatschner -* Samuel Loury -* Peter Michely -* Panashe M. Fundira -* Lucas Hoffmann -* Larry Hynes -* Kirill A. Shutemov -* Johannes Altmanninger -* Jeremy Kaplan -* Ismail -* Edgar Hipp -* Daryl Finlay -* arza -* adam -* Samir Benmendil -* Regina Hug -* Penaz -* Matthias Lisin -* Mathias Fussenegger -* Marcelo Santos -* Marcel Schilling -* Joel Bradshaw -* Jean-Louis Fuchs -* Franz Fellner -* Eric Drechsel -* zwarag -* xd1le -* rmortens -* oniondreams -* issue -* haxwithaxe -* evan -* dylan araps -* caveman -* addictedtoflames -* Xitian9 -* Vasilij Schneidermann -* Tomas Orsava -* Tom Janson -* Tobias Werth -* Tim Harder -* Thiago Barroso Perrotta -* Steve Peak -* Sorokin Alexei -* Simon Désaulniers -* Rok Mandeljc -* Noah Huesser -* Moez Bouhlel -* MikeinRealLife -* Lazlow Carmichael -* Kevin Wang -* Ján Kobezda -* Justin Partain -* Johannes Martinsson -* Jean-Christophe Petkovich -* Helen Sherwood-Taylor -* HalosGhost -* Gregor Pohl -* Eivind Uggedal -* Dietrich Daroch -* Derek Sivers -* Daniel Lu -* Daniel Jakots -* Daniel Hahler -* Arseniy Seroka -* Anton Grensjö -* Andy Balaam -* Andreas Fischer -* Amos Bird -* Akselmo -// QUTE_AUTHORS_END - -The following people have contributed graphics: +Additionally, the following people have contributed graphics: * Jad/link:http://yelostudio.com[yelo] (new icon) * WOFall (original icon) * regines (key binding cheatsheet) -Thanks / Similar projects -------------------------- +Also, thanks to everyone who contributed to one of qutebrowser's +link:doc/backers.asciidoc[crowdfunding campaigns]! -Many projects with a similar goal as qutebrowser exist: - -* http://portix.bitbucket.org/dwb/[dwb] (C, GTK+ with WebKit1, currently -http://www.reddit.com/r/linux/comments/2huqbc/dwb_abandoned/[unmaintained] - -main inspiration for qutebrowser) -* https://github.com/fanglingsu/vimb[vimb] (C, GTK+ with WebKit1, active) -* http://sourceforge.net/p/vimprobable/wiki/Home/[vimprobable] (C, GTK+ with -WebKit1, dead) -* http://surf.suckless.org/[surf] (C, GTK+ with WebKit1, active) -* https://mason-larobina.github.io/luakit/[luakit] (C/Lua, GTK+ with -WebKit1, not very active) -* http://pwmt.org/projects/jumanji/[jumanji] (C, GTK+ with WebKit1, not very -active) -* http://www.uzbl.org/[uzbl] (C, GTK+ with WebKit1/WebKit2, active) -* http://conkeror.org/[conkeror] (Javascript, Emacs-like, XULRunner/Gecko, -active) -* https://github.com/AeroNotix/lispkit[lispkit] (quite new, lisp, GTK+ with -WebKit, active) -* http://www.vimperator.org/[Vimperator] (Firefox addon) -* http://5digits.org/pentadactyl/[Pentadactyl] (Firefox addon) -* https://github.com/akhodakivskiy/VimFx[VimFx] (Firefox addon) -* https://github.com/1995eaton/chromium-vim[cVim] (Chrome/Chromium addon) -* http://vimium.github.io/[vimium] (Chrome/Chromium addon) -* https://chrome.google.com/webstore/detail/vichrome/gghkfhpblkcmlkmpcpgaajbbiikbhpdi?hl=en[ViChrome] (Chrome/Chromium addon) -* https://github.com/jinzhu/vrome[Vrome] (Chrome/Chromium addon) +Similar projects +---------------- +Many projects with a similar goal as qutebrowser exist. Most of them were inspirations for qutebrowser in some way, thanks for that! -Thanks as well to the following projects and people for helping me with -problems and helpful hints: +Active +~~~~~~ -* http://eric-ide.python-projects.org/[eric5] / Detlev Offenbach -* https://code.google.com/p/devicenzo/[devicenzo] -* portix -* seir -* nitroxleecher +* https://github.com/fanglingsu/vimb[vimb] (C, GTK+ with WebKit2) +* https://mason-larobina.github.io/luakit/[luakit] (C/Lua, GTK+ with WebKit2) +* http://surf.suckless.org/[surf] (C, GTK+ with WebKit1/WebKit2) +* http://www.uzbl.org/[uzbl] (C, GTK+ with WebKit1/WebKit2) +* Chrome/Chromium addons: + https://github.com/1995eaton/chromium-vim[cVim], + http://vimium.github.io/[Vimium], + https://github.com/brookhong/Surfingkeys[Surfingkeys], + https://github.com/lusakasa/saka-key[Saka Key] +* Firefox addons (based on WebExtensions): + https://addons.mozilla.org/en-GB/firefox/addon/vimium-ff/[Vimium-FF] (experimental), + https://github.com/lusakasa/saka-key[Saka Key] -Also, thanks to: +Inactive +~~~~~~~~ -* Everyone contributing to the link:doc/backers.asciidoc[crowdfunding]. -* Everyone who had the patience to test qutebrowser before v0.1. -* Everyone triaging/fixing my bugs in the -https://bugreports.qt.io/secure/Dashboard.jspa[Qt bugtracker] -* Everyone answering my questions on http://stackoverflow.com/[Stack Overflow] -and in IRC. -* All the projects which were a great help while developing qutebrowser. +* http://portix.bitbucket.org/dwb/[dwb] (C, GTK+ with WebKit1, +http://www.reddit.com/r/linux/comments/2huqbc/dwb_abandoned/[unmaintained] - +main inspiration for qutebrowser) +* http://sourceforge.net/p/vimprobable/wiki/Home/[vimprobable] (C, GTK+ with + WebKit1) +* http://pwmt.org/projects/jumanji/[jumanji] (C, GTK+ with WebKit1) +* http://conkeror.org/[conkeror] (Javascript, Emacs-like, XULRunner/Gecko) +* Firefox addons (not based on WebExtensions or no recent activity): + http://www.vimperator.org/[Vimperator], + http://5digits.org/pentadactyl/[Pentadactyl] + https://github.com/akhodakivskiy/VimFx[VimFx] (Firefox addon), + https://github.com/shinglyu/QuantumVim[QuantumVim] +* Chrome/Chromium addons: + https://chrome.google.com/webstore/detail/vichrome/gghkfhpblkcmlkmpcpgaajbbiikbhpdi?hl=en[ViChrome], + https://github.com/jinzhu/vrome[Vrome] License ------- diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 05bf3ad93..2c77b0b76 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -1,5 +1,5 @@ // DO NOT EDIT THIS FILE DIRECTLY! -// It is autogenerated from docstrings by running: +// It is autogenerated by running: // $ python3 scripts/dev/src2asciidoc.py = Commands diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index b038d462d..9ba64aecc 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -1,5 +1,5 @@ // DO NOT EDIT THIS FILE DIRECTLY! -// It is autogenerated from docstrings by running: +// It is autogenerated by running: // $ python3 scripts/dev/src2asciidoc.py = Settings diff --git a/scripts/asciidoc2html.py b/scripts/asciidoc2html.py index 39695bd23..6c7fdaf6e 100755 --- a/scripts/asciidoc2html.py +++ b/scripts/asciidoc2html.py @@ -280,8 +280,6 @@ def main(colors=False): "asciidoc.py. If not given, it's searched in PATH.", nargs=2, required=False, metavar=('PYTHON', 'ASCIIDOC')) - parser.add_argument('--no-authors', help=argparse.SUPPRESS, - action='store_true') args = parser.parse_args() try: os.mkdir('qutebrowser/html/doc') diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index 6e148c088..80a45026a 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -4,7 +4,6 @@ if [[ $DOCKER ]]; then docker run --privileged -v $PWD:/outside -e QUTE_BDD_WEBENGINE=$QUTE_BDD_WEBENGINE -e DOCKER=$DOCKER qutebrowser/travis:$DOCKER else args=() - [[ $TESTENV == docs ]] && args=('--no-authors') [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb') tox -e $TESTENV -- "${args[@]}" diff --git a/scripts/dev/src2asciidoc.py b/scripts/dev/src2asciidoc.py index a202edeb3..5fe6af6ba 100755 --- a/scripts/dev/src2asciidoc.py +++ b/scripts/dev/src2asciidoc.py @@ -27,7 +27,6 @@ import shutil import os.path import inspect import subprocess -import collections import tempfile import argparse @@ -44,7 +43,7 @@ from qutebrowser.utils import docutils, usertypes FILE_HEADER = """ // DO NOT EDIT THIS FILE DIRECTLY! -// It is autogenerated from docstrings by running: +// It is autogenerated by running: // $ python3 scripts/dev/src2asciidoc.py """.lstrip() @@ -421,32 +420,6 @@ def generate_settings(filename): _generate_setting_section(f, sectname, sect) -def _get_authors(): - """Get a list of authors based on git commit logs.""" - corrections = { - 'binix': 'sbinix', - 'Averrin': 'Alexey "Averrin" Nabrodov', - 'Alexey Nabrodov': 'Alexey "Averrin" Nabrodov', - 'Michael': 'Halfwit', - 'Error 800': 'error800', - 'larryhynes': 'Larry Hynes', - 'Daniel': 'Daniel Schadt', - 'Alexey Glushko': 'haitaka', - 'Corentin Jule': 'Corentin Julé', - 'Claire C.C': 'Claire Cavanaugh', - 'Rahid': 'Maciej Wołczyk', - 'Fritz V155 Reichwald': 'Fritz Reichwald', - 'Spreadyy': 'sandrosc', - } - ignored = ['pyup-bot'] - commits = subprocess.check_output(['git', 'log', '--format=%aN']) - authors = [corrections.get(author, author) - for author in commits.decode('utf-8').splitlines() - if author not in ignored] - cnt = collections.Counter(authors) - return sorted(cnt, key=lambda k: (cnt[k], k), reverse=True) - - def _format_block(filename, what, data): """Format a block in a file. @@ -493,12 +466,6 @@ def _format_block(filename, what, data): shutil.move(tmpname, filename) -def regenerate_authors(filename): - """Re-generate the authors inside README based on the commits made.""" - data = ['* {}\n'.format(author) for author in _get_authors()] - _format_block(filename, 'authors', data) - - def regenerate_manpage(filename): """Update manpage OPTIONS using an argparse parser.""" # pylint: disable=protected-access @@ -544,9 +511,6 @@ def main(): generate_settings('doc/help/settings.asciidoc') print("Generating command help...") generate_commands('doc/help/commands.asciidoc') - if '--no-authors' not in sys.argv: - print("Generating authors in README...") - regenerate_authors('README.asciidoc') if '--cheatsheet' in sys.argv: print("Regenerating cheatsheet .pngs") regenerate_cheatsheet() From 03a0bfddddf5cdcd54571bac5f5c5e58a4c7e978 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 21 Jul 2017 13:36:17 +0200 Subject: [PATCH 276/337] Some more README improvements --- README.asciidoc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.asciidoc b/README.asciidoc index 1604b85ec..ef517ba5b 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -96,15 +96,16 @@ Requirements The following software and libraries are required to run qutebrowser: * http://www.python.org/[Python] 3.4 or newer (3.6 recommended) - note that - Python 3.4 support will be dropped soon. + support for Python 3.4 + https://github.com/qutebrowser/qutebrowser/issues/2742[will be dropped soon]. * http://qt.io/[Qt] 5.2.0 or newer (5.9 recommended) - note that support for Qt < 5.7.1 will be dropped soon. * QtWebKit (old or link:https://github.com/annulen/webkit/wiki[reloaded]/NG) or QtWebEngine. Note that support for legacy QtWebKit (before 5.212) will be dropped soon. * http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.2.0 or newer -(5.9 recommended) for Python 3. Note that support for PyQt < 5.7 will be dropped -soon. + (5.9 recommended) for Python 3. Note that support for PyQt < 5.7 will be + dropped soon. * https://pypi.python.org/pypi/setuptools/[pkg_resources/setuptools] * http://fdik.org/pyPEG/[pyPEG2] * http://jinja.pocoo.org/[jinja2] @@ -165,24 +166,24 @@ Most of them were inspirations for qutebrowser in some way, thanks for that! Active ~~~~~~ -* https://github.com/fanglingsu/vimb[vimb] (C, GTK+ with WebKit2) -* https://mason-larobina.github.io/luakit/[luakit] (C/Lua, GTK+ with WebKit2) +* https://fanglingsu.github.io/vimb/[vimb] (C, GTK+ with WebKit2) +* https://luakit.github.io/luakit/[luakit] (C/Lua, GTK+ with WebKit2) * http://surf.suckless.org/[surf] (C, GTK+ with WebKit1/WebKit2) * http://www.uzbl.org/[uzbl] (C, GTK+ with WebKit1/WebKit2) * Chrome/Chromium addons: https://github.com/1995eaton/chromium-vim[cVim], http://vimium.github.io/[Vimium], https://github.com/brookhong/Surfingkeys[Surfingkeys], - https://github.com/lusakasa/saka-key[Saka Key] + http://saka-key.lusakasa.com/[Saka Key] * Firefox addons (based on WebExtensions): https://addons.mozilla.org/en-GB/firefox/addon/vimium-ff/[Vimium-FF] (experimental), - https://github.com/lusakasa/saka-key[Saka Key] + http://saka-key.lusakasa.com/[Saka Key] Inactive ~~~~~~~~ -* http://portix.bitbucket.org/dwb/[dwb] (C, GTK+ with WebKit1, -http://www.reddit.com/r/linux/comments/2huqbc/dwb_abandoned/[unmaintained] - +* https://bitbucket.org/portix/dwb[dwb] (C, GTK+ with WebKit1, +https://bitbucket.org/portix/dwb/pull-requests/22/several-cleanups-to-increase-portability/diff[unmaintained] - main inspiration for qutebrowser) * http://sourceforge.net/p/vimprobable/wiki/Home/[vimprobable] (C, GTK+ with WebKit1) From f1d4f693bbae918428de440e9773f19b80bf13ee Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 21 Jul 2017 13:40:04 +0200 Subject: [PATCH 277/337] Whoops - hopefully final README fix --- README.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.asciidoc b/README.asciidoc index ef517ba5b..dcf478aa2 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -191,8 +191,8 @@ main inspiration for qutebrowser) * http://conkeror.org/[conkeror] (Javascript, Emacs-like, XULRunner/Gecko) * Firefox addons (not based on WebExtensions or no recent activity): http://www.vimperator.org/[Vimperator], - http://5digits.org/pentadactyl/[Pentadactyl] - https://github.com/akhodakivskiy/VimFx[VimFx] (Firefox addon), + http://5digits.org/pentadactyl/[Pentadactyl], + https://github.com/akhodakivskiy/VimFx[VimFx], https://github.com/shinglyu/QuantumVim[QuantumVim] * Chrome/Chromium addons: https://chrome.google.com/webstore/detail/vichrome/gghkfhpblkcmlkmpcpgaajbbiikbhpdi?hl=en[ViChrome], From 33a9c8cce6082ba1a158faaa5b409133dddf644a Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 21 Jul 2017 07:59:22 -0400 Subject: [PATCH 278/337] Add listcategory to perfect_files. --- scripts/dev/check_coverage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py index e63bb821f..c9e3836b6 100644 --- a/scripts/dev/check_coverage.py +++ b/scripts/dev/check_coverage.py @@ -158,6 +158,8 @@ PERFECT_FILES = [ 'completion/models/urlmodel.py'), ('tests/unit/completion/test_histcategory.py', 'completion/models/histcategory.py'), + ('tests/unit/completion/test_listcategory.py', + 'completion/models/listcategory.py'), ] From 5bea9c779408409e523d9e64f15e3fe9ed0a5763 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 21 Jul 2017 15:16:54 +0200 Subject: [PATCH 279/337] Some more doc improvements --- CONTRIBUTING.asciidoc | 6 ++++++ README.asciidoc | 14 +++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.asciidoc b/CONTRIBUTING.asciidoc index b7a13eaf8..fc039ad03 100644 --- a/CONTRIBUTING.asciidoc +++ b/CONTRIBUTING.asciidoc @@ -5,6 +5,12 @@ The Compiler :data-uri: :toc: +IMPORTANT: I'm currently (July 2017) more busy than usual until September, +because of exams coming up. In addition to that, a new config system is coming +which will conflict with many non-trivial contributions. Because of that, please +refrain from contributing new features until then. If you're reading this note +after mid-September, please open an issue. + I `<3` footnote:[Of course, that says `<3` in HTML.] contributors! This document contains guidelines for contributing to qutebrowser, as well as diff --git a/README.asciidoc b/README.asciidoc index dcf478aa2..a3b4918e4 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -98,11 +98,15 @@ The following software and libraries are required to run qutebrowser: * http://www.python.org/[Python] 3.4 or newer (3.6 recommended) - note that support for Python 3.4 https://github.com/qutebrowser/qutebrowser/issues/2742[will be dropped soon]. -* http://qt.io/[Qt] 5.2.0 or newer (5.9 recommended) - note that support for Qt - < 5.7.1 will be dropped soon. -* QtWebKit (old or link:https://github.com/annulen/webkit/wiki[reloaded]/NG) or - QtWebEngine. Note that support for legacy QtWebKit (before 5.212) will be - dropped soon. +* http://qt.io/[Qt] 5.2.0 or newer (5.9 recommended - note that support for Qt + < 5.7.1 will be dropped soon) with the following modules: + - QtCore / qtbase + - QtQuick (part of qtbase in some distributions) + - QtSQL (part of qtbase in some distributions) + - QtWebEngine, or + - QtWebKit (old or link:https://github.com/annulen/webkit/wiki[reloaded]/NG). + Note that support for legacy QtWebKit (before 5.212) will be + dropped soon. * http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.2.0 or newer (5.9 recommended) for Python 3. Note that support for PyQt < 5.7 will be dropped soon. From de0b50eaf7ba9b2821d77632af0479deb41e5a8e Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 21 Jul 2017 15:39:36 +0200 Subject: [PATCH 280/337] Update docs --- doc/help/commands.asciidoc | 10 ++++++++++ doc/help/settings.asciidoc | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 2c77b0b76..4fca0a272 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -1553,6 +1553,7 @@ These commands are mainly intended for debugging. They are hidden if qutebrowser |<>|Clear remembered SSL error answers. |<>|Show the debugging console. |<>|Crash for debugging purposes. +|<>|Dump the history to a file in the old pre-SQL format. |<>|Dump the current page's content to a file. |<>|Change the number of log lines to be stored in RAM. |<>|Change the log filter for console logging. @@ -1587,6 +1588,15 @@ Crash for debugging purposes. ==== positional arguments * +'typ'+: either 'exception' or 'segfault'. +[[debug-dump-history]] +=== debug-dump-history +Syntax: +:debug-dump-history 'dest'+ + +Dump the history to a file in the old pre-SQL format. + +==== positional arguments +* +'dest'+: Where to write the file to. + [[debug-dump-page]] === debug-dump-page Syntax: +:debug-dump-page [*--plain*] 'dest'+ diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 9ba64aecc..92bc918e1 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -899,7 +899,7 @@ How many URLs to show in the web history. 0: no history / -1: unlimited -Default: +pass:[1000]+ +Default: +pass:[-1]+ [[completion-quick-complete]] === quick-complete From 66602978710966c222ae41236ee13bb39327a7e0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 21 Jul 2017 17:10:03 +0200 Subject: [PATCH 281/337] Fix new completion with web-history-max-items set to 0 We get no last_atime limit at all otherwise: qutebrowser.misc.sql.SqlException: Failed to prepare query "SELECT url, title, strftime('%Y-%m-%d', last_atime, 'unixepoch', 'localtime') FROM CompletionHistory WHERE (url LIKE :pat escape '\' or title LIKE :pat escape '\') AND last_atime >= ORDER BY last_atime DESC": "near "ORDER": syntax error Unable to execute statement" --- qutebrowser/completion/models/histcategory.py | 9 +++++---- tests/unit/completion/test_histcategory.py | 7 ++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index fa8443a60..c87ba026c 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -60,7 +60,7 @@ class HistoryCategory(QSqlQueryModel): def _atime_expr(self): """If max_items is set, return an expression to limit the query.""" max_items = config.get('completion', 'web-history-max-items') - if max_items < 0: + if max_items <= 0: return '' min_atime = sql.Query(' '.join([ @@ -83,9 +83,10 @@ class HistoryCategory(QSqlQueryModel): # treat spaces as wildcards to match any of the typed words pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) - with debug.log_time('sql', 'Running completion query'): - self._query.run(pat=pattern) - self.setQuery(self._query) + if config.get('completion', 'web-history-max-items') != 0: + with debug.log_time('sql', 'Running completion query'): + self._query.run(pat=pattern) + self.setQuery(self._query) def delete_cur_item(self, index): """Delete the row at the given index.""" diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index 0b5fcb915..bfd9d4aa5 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -117,7 +117,12 @@ def test_set_pattern(pattern, before, after, model_validator, hist): ], [ ('b', 'b', '2017-06-16'), ('c', 'c', '2017-05-16'), - ]) + ]), + (0, [ + ('a', 'a', '2017-04-16'), + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ], []), ]) def test_sorting(max_items, before, after, model_validator, hist, config_stub): """Validate the filtering and sorting results of set_pattern.""" From 544094ba722ad3c9b2ee53977d55cdf4805cc00f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 21 Jul 2017 17:55:47 +0200 Subject: [PATCH 282/337] Use simpler way of preventing History completion --- qutebrowser/completion/models/histcategory.py | 12 +++++++----- qutebrowser/completion/models/urlmodel.py | 6 ++++-- tests/unit/completion/test_histcategory.py | 7 +------ tests/unit/completion/test_models.py | 10 ++++++++++ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index c87ba026c..612eb0bf4 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -60,7 +60,10 @@ class HistoryCategory(QSqlQueryModel): def _atime_expr(self): """If max_items is set, return an expression to limit the query.""" max_items = config.get('completion', 'web-history-max-items') - if max_items <= 0: + # HistoryCategory should not be added to the completion in that case. + assert max_items != 0 + + if max_items < 0: return '' min_atime = sql.Query(' '.join([ @@ -83,10 +86,9 @@ class HistoryCategory(QSqlQueryModel): # treat spaces as wildcards to match any of the typed words pattern = re.sub(r' +', '%', pattern) pattern = '%{}%'.format(pattern) - if config.get('completion', 'web-history-max-items') != 0: - with debug.log_time('sql', 'Running completion query'): - self._query.run(pat=pattern) - self.setQuery(self._query) + with debug.log_time('sql', 'Running completion query'): + self._query.run(pat=pattern) + self.setQuery(self._query) def delete_cur_item(self, index): """Delete the row at the given index.""" diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 0c5fbeacc..fbb9661ec 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -22,6 +22,7 @@ from qutebrowser.completion.models import (completionmodel, listcategory, histcategory) from qutebrowser.utils import log, objreg +from qutebrowser.config import config _URLCOL = 0 @@ -65,6 +66,7 @@ def url(): model.add_category(listcategory.ListCategory( 'Bookmarks', bookmarks, delete_func=_delete_bookmark)) - hist_cat = histcategory.HistoryCategory(delete_func=_delete_history) - model.add_category(hist_cat) + if config.get('completion', 'web-history-max-items') != 0: + hist_cat = histcategory.HistoryCategory(delete_func=_delete_history) + model.add_category(hist_cat) return model diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index bfd9d4aa5..0b5fcb915 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -117,12 +117,7 @@ def test_set_pattern(pattern, before, after, model_validator, hist): ], [ ('b', 'b', '2017-06-16'), ('c', 'c', '2017-05-16'), - ]), - (0, [ - ('a', 'a', '2017-04-16'), - ('b', 'b', '2017-06-16'), - ('c', 'c', '2017-05-16'), - ], []), + ]) ]) def test_sorting(max_items, before, after, model_validator, hist, config_stub): """Validate the filtering and sorting results of set_pattern.""" diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 5b632eb3a..83c3d3e7d 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -396,6 +396,16 @@ def test_url_completion_delete_history(qtmodeltester, assert 'https://python.org' not in web_history_populated +def test_url_completion_zero_limit(config_stub, web_history, quickmarks, + bookmarks): + """Make sure there's no history if the limit was set to zero.""" + config_stub.data['completion']['web-history-max-items'] = 0 + model = urlmodel.url() + model.set_pattern('') + category = model.index(2, 0) # "History" normally + assert model.data(category) is None + + def test_session_completion(qtmodeltester, session_manager_stub): session_manager_stub.sessions = ['default', '1', '2'] model = miscmodels.session() From 118a7942a521ee711608ea5da84a2b975c20c228 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 21 Jul 2017 18:30:12 +0200 Subject: [PATCH 283/337] Add maximum bound for web-history-max-items sqlite can't handle values bigger than uint64_t for LIMIT. --- qutebrowser/config/configdata.py | 2 +- tests/unit/completion/test_histcategory.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/qutebrowser/config/configdata.py b/qutebrowser/config/configdata.py index 23d3efb67..92878d931 100644 --- a/qutebrowser/config/configdata.py +++ b/qutebrowser/config/configdata.py @@ -503,7 +503,7 @@ def data(readonly=False): "0: no history / -1: unlimited"), ('web-history-max-items', - SettingValue(typ.Int(minval=-1), '-1'), + SettingValue(typ.Int(minval=-1, maxval=MAXVALS['int64']), '-1'), "How many URLs to show in the web history.\n\n" "0: no history / -1: unlimited"), diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index 0b5fcb915..c53c86ee1 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -110,6 +110,15 @@ def test_set_pattern(pattern, before, after, model_validator, hist): ('c', 'c', '2017-05-16'), ('a', 'a', '2017-04-16'), ]), + (2 ** 63 - 1, [ # Maximum value sqlite can handle for LIMIT + ('a', 'a', '2017-04-16'), + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ], [ + ('b', 'b', '2017-06-16'), + ('c', 'c', '2017-05-16'), + ('a', 'a', '2017-04-16'), + ]), (2, [ ('a', 'a', '2017-04-16'), ('b', 'b', '2017-06-16'), From f9dc31e464661dbed73f0b90fab0c9ab37d844ef Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sat, 22 Jul 2017 10:22:57 +0200 Subject: [PATCH 284/337] Add subreddit to README --- README.asciidoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.asciidoc b/README.asciidoc index a3b4918e4..f7ad0f64d 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -71,6 +71,9 @@ There's also a https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-ann at mailto:qutebrowser-announce@lists.qutebrowser.org[] (the announcements also get sent to the general qutebrowser@ list). +If you're a reddit user, there's a +https://www.reddit.com/r/qutebrowser/[/r/qutebrowser] subreddit there. + Contributions / Bugs -------------------- From 27dfc72012c896e8ac39eb4ded8689e9b2162f08 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Wed, 19 Jul 2017 23:11:39 -0700 Subject: [PATCH 285/337] Restructure minimum tab size behavior --- qutebrowser/mainwindow/tabwidget.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 4756d231e..c70d4f4ca 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -503,10 +503,6 @@ class TabBar(QTabBar): # We return it directly rather than setting `size' because we don't # want to ensure it's valid in this special case. return QSize() - elif self.count() * minimum_size.width() > self.width(): - # If we don't have enough space, we return the minimum size so we - # get scroll buttons as soon as needed. - size = minimum_size else: tab_width_pinned_conf = config.get('tabs', 'pinned-width') @@ -517,21 +513,19 @@ class TabBar(QTabBar): no_pinned_count = self.count() - self.pinned_count pinned_width = tab_width_pinned_conf * self.pinned_count - # Prevent any tabs from being smaller than the min size - no_pinned_width = max(self.width() - pinned_width, - minimum_size.width() * no_pinned_count) + no_pinned_width = self.width() - pinned_width if pinned: width = tab_width_pinned_conf else: - # If we *do* have enough space, tabs should occupy the whole - # window width. If there are pinned tabs their size will be - # subtracted from the total window width. - # During shutdown the self.count goes down, - # but the self.pinned_count not - this generates some odd + # Tabs should attempt to occupy the whole window width. If + # there are pinned tabs their size will be subtracted from the + # total window width. During shutdown the self.count goes + # down, but the self.pinned_count not - this generates some odd # behavior. To avoid this we compare self.count against - # self.pinned_count. + # self.pinned_count. If we end up having too little space, we + # set the minimum size below. if self.pinned_count > 0 and no_pinned_count > 0: width = no_pinned_width / no_pinned_count else: @@ -543,6 +537,10 @@ class TabBar(QTabBar): index < no_pinned_width % no_pinned_count): width += 1 + # If we don't have enough space, we return the minimum size so we + # get scroll buttons as soon as needed. + width = max(width, minimum_size.width()) + size = QSize(width, height) qtutils.ensure_valid(size) return size From bc21904fef99c0a131a6083abdff805b9cede2b3 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 22 Jul 2017 15:12:31 -0400 Subject: [PATCH 286/337] Fix completion-item-del on undeletable item. Even though no item was deleted, it was manipulating the completion model because beginRemoveRows was called before the exception was raised. This fixes that problem by moving the removal logic (and delete_func check) into the parent model, so it can check whether deletion is possible before calling beginRemoveRows. Fixes #2839. --- .../completion/models/completionmodel.py | 10 +++++++++- qutebrowser/completion/models/histcategory.py | 9 ++------- qutebrowser/completion/models/listcategory.py | 9 --------- tests/unit/completion/test_completionmodel.py | 18 +++++++++++++++++ tests/unit/completion/test_histcategory.py | 20 +++++++------------ tests/unit/completion/test_listcategory.py | 20 ------------------- 6 files changed, 36 insertions(+), 50 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 3e48076e5..398673200 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -22,6 +22,7 @@ from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel from qutebrowser.utils import log, qtutils +from qutebrowser.commands import cmdexc class CompletionModel(QAbstractItemModel): @@ -219,6 +220,13 @@ class CompletionModel(QAbstractItemModel): parent = index.parent() cat = self._cat_from_idx(parent) assert cat, "CompletionView sent invalid index for deletion" + if not cat.delete_func: + raise cmdexc.CommandError("Cannot delete this item.") + + data = [cat.data(cat.index(index.row(), i)) + for i in range(cat.columnCount())] + cat.delete_func(data) + self.beginRemoveRows(parent, index.row(), index.row()) - cat.delete_cur_item(cat.index(index.row(), 0)) + cat.removeRow(index.row(), QModelIndex()) self.endRemoveRows() diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index 612eb0bf4..56b7c3819 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -90,13 +90,8 @@ class HistoryCategory(QSqlQueryModel): self._query.run(pat=pattern) self.setQuery(self._query) - def delete_cur_item(self, index): - """Delete the row at the given index.""" - if not self.delete_func: - raise cmdexc.CommandError("Cannot delete this item.") - data = [self.data(index.sibling(index.row(), i)) - for i in range(self.columnCount())] - self.delete_func(data) + def removeRow(self, _row, _col): + """Re-run sql query to respond to a row removal.""" # re-run query to reload updated table with debug.log_time('sql', 'Re-running completion query post-delete'): self._query.run() diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index 187ebcad6..5f537d4d7 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -91,12 +91,3 @@ class ListCategory(QSortFilterProxyModel): return False else: return left < right - - def delete_cur_item(self, index): - """Delete the row at the given index.""" - if not self.delete_func: - raise cmdexc.CommandError("Cannot delete this item.") - data = [self.data(index.sibling(index.row(), i)) - for i in range(self.columnCount())] - self.delete_func(data) - self.removeRow(index.row(), QModelIndex()) diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 4d1d3f123..9e73e533a 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -28,6 +28,7 @@ from PyQt5.QtCore import QModelIndex from qutebrowser.completion.models import completionmodel, listcategory from qutebrowser.utils import qtutils +from qutebrowser.commands import cmdexc @hypothesis.given(strategies.lists(min_size=0, max_size=3, @@ -92,8 +93,25 @@ def test_delete_cur_item(): func.assert_called_once_with(['foo', 'bar']) +def test_delete_cur_item_no_func(): + callback = mock.Mock(spec=[]) + model = completionmodel.CompletionModel() + cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=None) + model.rowsAboutToBeRemoved.connect(callback) + model.rowsRemoved.connect(callback) + model.add_category(cat) + parent = model.index(0, 0) + with pytest.raises(cmdexc.CommandError): + model.delete_cur_item(model.index(0, 0, parent)) + assert not callback.called + + def test_delete_cur_item_no_cat(): """Test completion_item_del with no selected category.""" + callback = mock.Mock(spec=[]) model = completionmodel.CompletionModel() + model.rowsAboutToBeRemoved.connect(callback) + model.rowsRemoved.connect(callback) with pytest.raises(qtutils.QtValueError): model.delete_cur_item(QModelIndex()) + assert not callback.called diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index c53c86ee1..7044f1281 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -23,6 +23,7 @@ import unittest.mock import datetime import pytest +from PyQt5.QtCore import QModelIndex from qutebrowser.misc import sql from qutebrowser.completion.models import histcategory @@ -140,20 +141,13 @@ def test_sorting(max_items, before, after, model_validator, hist, config_stub): model_validator.validate(after) -def test_delete_cur_item(hist): +def test_remove_row(hist, model_validator): hist.insert({'url': 'foo', 'title': 'Foo'}) hist.insert({'url': 'bar', 'title': 'Bar'}) - func = unittest.mock.Mock(spec=[]) - cat = histcategory.HistoryCategory(delete_func=func) - cat.set_pattern('') - cat.delete_cur_item(cat.index(0, 0)) - func.assert_called_with(['foo', 'Foo', '']) - - -def test_delete_cur_item_no_func(hist): - hist.insert({'url': 'foo', 'title': 1}) - hist.insert({'url': 'bar', 'title': 2}) cat = histcategory.HistoryCategory() + model_validator.set_model(cat) cat.set_pattern('') - with pytest.raises(cmdexc.CommandError, match='Cannot delete this item'): - cat.delete_cur_item(cat.index(0, 0)) + hist.delete('url', 'foo') + # histcategory does not care which index was removed, it just regenerates + cat.removeRow(QModelIndex(), QModelIndex()) + model_validator.validate([('bar', 'Bar', '')]) diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index 3b1c1478a..df1d1de5f 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -51,23 +51,3 @@ def test_set_pattern(pattern, before, after, model_validator): model_validator.set_model(cat) cat.set_pattern(pattern) model_validator.validate(after) - - -def test_delete_cur_item(model_validator): - func = mock.Mock(spec=[]) - cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')], - delete_func=func) - model_validator.set_model(cat) - idx = cat.index(0, 0) - cat.delete_cur_item(idx) - func.assert_called_once_with(['a', 'b']) - model_validator.validate([('c', 'd')]) - - -def test_delete_cur_item_no_func(model_validator): - cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')]) - model_validator.set_model(cat) - idx = cat.index(0, 0) - with pytest.raises(cmdexc.CommandError, match="Cannot delete this item."): - cat.delete_cur_item(idx) - model_validator.validate([('a', 'b'), ('c', 'd')]) From b61691684ec4f3a6c2f4682dd087bcd142a2bd63 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 22 Jul 2017 18:06:16 -0400 Subject: [PATCH 287/337] Clear selection when setting completion pattern. It doesn't make sense to have an active selection while you are filtering by entering text. You should be in one of two states: 1. Tabbing through completions (valid selection) 2. Entering a filter pattern (invalid selection) Fixes #2843, where a crash would occur after the following: 1. tab to an item other than the first 2. 3. re-type last character 4. This would try to delete an out of range index. --- qutebrowser/completion/completionwidget.py | 1 + tests/unit/completion/test_completionwidget.py | 1 + 2 files changed, 2 insertions(+) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 6e1e51680..39fe16e86 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -297,6 +297,7 @@ class CompletionView(QTreeView): self.pattern = pattern with debug.log_time(log.completion, 'Set pattern {}'.format(pattern)): self.model().set_pattern(pattern) + self.selectionModel().clear() self._maybe_update_geometry() self._maybe_show() diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 207e557a8..22d150fd8 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -87,6 +87,7 @@ def test_set_pattern(completionview): completionview.set_model(model) completionview.set_pattern('foo') model.set_pattern.assert_called_with('foo') + assert not completionview.selectionModel().currentIndex().isValid() def test_set_pattern_no_model(completionview): From 00be9e3c7fb269aa26942f6fcaf374c9b88cc8f6 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 22 Jul 2017 18:09:10 -0400 Subject: [PATCH 288/337] Remove obsolete TODO. New aliases will now show up without a signal, as completions are generated on-demand. --- qutebrowser/completion/completionwidget.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 39fe16e86..65f1de76c 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -109,8 +109,6 @@ class CompletionView(QTreeView): super().__init__(parent) self.pattern = '' self._win_id = win_id - # FIXME handle new aliases. - # objreg.get('config').changed.connect(self.init_command_completion) objreg.get('config').changed.connect(self._on_config_changed) self._active = False From 353f86488ae190c0d1a357ac0f6f9616fead3234 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 23 Jul 2017 21:11:33 +0200 Subject: [PATCH 289/337] Disallow :spawn -u -d --- qutebrowser/browser/commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 75ef17c08..f3f71d910 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1160,6 +1160,7 @@ class CommandDispatcher: detach: Whether the command should be detached from qutebrowser. cmdline: The commandline to execute. """ + cmdutils.check_exclusive((userscript, detach), 'ud') try: cmd, *args = shlex.split(cmdline) except ValueError as e: From a08fd0fcb1f22449e0e71df6f1c99e87cc4eb03a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 23 Jul 2017 21:34:17 +0200 Subject: [PATCH 290/337] Fix error message with :spawn -d --- qutebrowser/misc/guiprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index e8d224f2e..f1917b523 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -155,7 +155,7 @@ class GUIProcess(QObject): self._started = True else: message.error("Error while spawning {}: {}.".format( - self._what, self._proc.error())) + self._what, ERROR_STRINGS[self._proc.error()])) def exit_status(self): return self._proc.exitStatus() From 7d10e4704641923e5e4db3104458283abf327f83 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 23 Jul 2017 21:35:32 +0200 Subject: [PATCH 291/337] Update changelog --- CHANGELOG.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 5e0ad3ee8..d28efa7e6 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -69,6 +69,8 @@ Fixes - Fixed printing on macOS. - Closing a pinned tab via mouse now also prompts for confirmation. - The "try again" button on error pages works correctly again. +- :spawn -u -d is now disallowed. +- :spawn -d shows error messages correctly now. v0.11.0 ------- From 56b4989f44073e4cc1baa9648ce8a13c69c996be Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 23 Jul 2017 22:10:50 +0200 Subject: [PATCH 292/337] Fix tests for QProcess changes --- qutebrowser/misc/guiprocess.py | 2 +- tests/end2end/features/spawn.feature | 5 ----- tests/unit/misc/test_guiprocess.py | 6 ++++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index f1917b523..95bfac79e 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -154,7 +154,7 @@ class GUIProcess(QObject): log.procs.debug("Process started.") self._started = True else: - message.error("Error while spawning {}: {}.".format( + message.error("Error while spawning {}: {}".format( self._what, ERROR_STRINGS[self._proc.error()])) def exit_status(self): diff --git a/tests/end2end/features/spawn.feature b/tests/end2end/features/spawn.feature index dc0485391..e2b5fdd5b 100644 --- a/tests/end2end/features/spawn.feature +++ b/tests/end2end/features/spawn.feature @@ -60,8 +60,3 @@ Feature: :spawn Scenario: Running :spawn with userscript that expects the stdin getting closed When I run :spawn -u (testdata)/userscripts/stdinclose.py Then the message "stdin closed" should be shown - - @posix - Scenario: Running :spawn -d with userscript that expects the stdin getting closed - When I run :spawn -d -u (testdata)/userscripts/stdinclose.py - Then the message "stdin closed" should be shown diff --git a/tests/unit/misc/test_guiprocess.py b/tests/unit/misc/test_guiprocess.py index 3101b7427..749031367 100644 --- a/tests/unit/misc/test_guiprocess.py +++ b/tests/unit/misc/test_guiprocess.py @@ -128,11 +128,13 @@ def test_start_detached_error(fake_proc, message_mock, caplog): """Test starting a detached process with ok=False.""" argv = ['foo', 'bar'] fake_proc._proc.startDetached.return_value = (False, 0) - fake_proc._proc.error.return_value = "Error message" + fake_proc._proc.error.return_value = QProcess.FailedToStart with caplog.at_level(logging.ERROR): fake_proc.start_detached(*argv) msg = message_mock.getmsg(usertypes.MessageLevel.error) - assert msg.text == "Error while spawning testprocess: Error message." + expected = ("Error while spawning testprocess: The process failed to " + "start.") + assert msg.text == expected def test_double_start(qtbot, proc, py_proc): From e402e37f125a0419b97b19780331c93d9acf596a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 23 Jul 2017 22:45:13 +0200 Subject: [PATCH 293/337] Work around segfault when using pdb --- qutebrowser/misc/crashsignal.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qutebrowser/misc/crashsignal.py b/qutebrowser/misc/crashsignal.py index 5f161312a..e51855af2 100644 --- a/qutebrowser/misc/crashsignal.py +++ b/qutebrowser/misc/crashsignal.py @@ -28,6 +28,11 @@ import functools import faulthandler import os.path import collections +try: + # WORKAROUND for segfaults when using pdb in pytest for some reason... + import readline # pylint: disable=unused-import +except ImportError: + pass from PyQt5.QtCore import (pyqtSlot, qInstallMessageHandler, QObject, QSocketNotifier, QTimer, QUrl) From 630e9ebd66dddbc6509147fe73f05a9376c77b8a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 23 Jul 2017 23:15:41 +0200 Subject: [PATCH 294/337] Remove old notes file --- doc/notes | 196 ------------------------------------------------------ 1 file changed, 196 deletions(-) delete mode 100644 doc/notes diff --git a/doc/notes b/doc/notes deleted file mode 100644 index 2261e3228..000000000 --- a/doc/notes +++ /dev/null @@ -1,196 +0,0 @@ -henk's thoughts -=============== - -1. Power to the user! Protect privacy! -Things the browser should only do with explicit consent from the user, if -applicable the user should be able to choose which protocol/host/port triplets -to white/blacklist: - -- load/run executable code, like js, flash, java applets, ... (think NoScript) -- requests to other domains, ports or using a different protocol than what the - user requested (think RequestPolicy) -- accept cookies -- storing/saving/caching things, e.g. open tabs ("session"), cookies, page - contents, browsing/download history, form data, ... -- send referrer -- disclose any (presence, type, version, settings, capabilities, etc.) - information about OS, browser, installed fonts, plugins, addons, etc. - -2. Be efficient! -I tend to leave a lot of tabs open and nobody can deny that some websites -simply suck, so the browser should, unless told otherwise by the user: - -- load tabs only when needed -- run code in tabs only when needed, i.e. when the tab is currently being - used/viewed (background tabs doing some JS magic even when they are not being - used can create a lot of unnecessary load on the machine) -- finish requests to the domain the user requested (e.g. www.example.org) - before doing any requests to other subdomains (e.g. images.example.org) and - finish those before doing requests to thirdparty domains (e.g. example.com) - -3. Be stable! -- one site should not make the complete browser crash, only that site's tab - - -Upstream Bugs -============= - -- Web inspector is blank unless .hide()/.show() is called. - Asked on SO: http://stackoverflow.com/q/23499159/2085149 - TODO: Report to PyQt/Qt - -- Report some other crashes - - -/u/angelic_sedition's thoughts -============================== - -Well support for greasemonkey scripts and bookmarklets/js (which was mentioned -in the arch forum post) would be a big addition. What I've usually missed when -using other vim-like browsers is things that allow for different settings and -key bindings for different contexts. With that implemented I think I could -switch to a lightweight browser (and believe me, I'd like to) for the most part -and only use firefox when I needed downthemall or something. - -For example, I have different bindings based on tab position that are reloaded -with a pentadactyl autocmd so that will take me to tab -1-10 if I'm in that range or 2-20 if I'm in that range. I have an autocmd that -will run on completed downloads that passes the file path to a script that will -open ranger in a floating window with that file cut (this is basically like -using ranger to save files instead of the crappy gui popup). - -I also have a few bindings based on tabgroups. Tabgroups are a firefox feature, -but I find them very useful for sorting things by topic so that only the tabs -I'm interested at the moment are visible. - -Pentadactyl has a feature it calls groups. You can create a group that will -activate for sites/urls that match a pattern with some regex support. This -allows me, for example, to set up different (more convenient) bindings for -zooming only on images. I'll never need use the equivalent of vim n (next text -search match), so I can bind that to zoom. This allows setting up custom -quickmarks/gotos using the same keys for different websites. For example, on -reddit I have different g(some key) bindings to go to different subreddits. -This can also be used to pass certain keys directly to the site (e.g. for use -with RES). For sites that don't have modifiable bindings, I can use this with -pentadactyl's feedkeys or xdotool to create my own custom bindings. I even have -a binding that will call out to bash script with different arguments depending -on the site to download an image or an image gallery depending on the site (in -some cases passing the url to some cli program). - -I've also noticed the lack of completion. For example, on "o" pentadactyl will -show sites (e.g. from history) that can be completed. I think I've been spoiled -by pentadactyl having completion for just about everything. - - -suckless surf ML post -===================== - -From: Ben Woolley -Date: Wed, 7 Jan 2015 18:29:25 -0800 - -Hi all, - -This patch is a bit of a beast for surf. It is intended to be applied after -the disk cache patch. It breaks some internal interfaces, so it could -conflict with other patches. - -I have been wanting a browser to implement a complete same-origin policy, -and have been investigating how to do this in various browsers for many -months. When I saw how surf opened new windows in a separate process, and -was so simple, I knew I could do it quickly. Over the last two weeks, I -have been developing this implementation on surf. - -The basic idea is to prevent browser-based tracking as you browse from site -to site, or origin to origin. By "origin" domain, I mean the "first-party" -domain, the domain normally in the location bar (of the typical browser -interface). Each origin domain effectively gets its own browser profile, -and a browser process only ever deals with one origin domain at a time. -This isolates origins vertically, preventing cookies, disk cache, memory -cache, and window.name vulnerabilities. Basically, all known -vulnerabilities that google and Mozilla cite as counter-examples when they -explain why they haven't disabled third-party cookies yet. - -When you are on msnbc.com, the tracking pixels will be stored in a cookie -file for msnbc.com. When you go to cnn.com, the tracking pixels will be -stored in a cookie file for cnn.com. You will not be tracked between them. -However, third-party cookies, and the caching of third party resources will -still work, but they will be isolated between origin domains. Instead of -blocking cookies and cache entries, they are "double-keyed", or *also* -keyed by origin. - -There is a unidirectional communication channel, however, from one origin -to the next, through navigation from one origin to the next. That is, the -query string is passed from one origin to the next, and may embed -identifiers. One example is an affiliate link that identifies where the -lead came from. I have implemented what I call "horizontal isolation", in -the form of an "Origin Crossing Gate". - -Whenever you follow a link to a new domain, or even are just redirected to -a new domain, a new window/tab is opened, and passed the referring origin -via -R. The page passed to -O, for example -O originprompt.html, is an HTML -page that is loaded in the new origin's context. That page tells you the -origin you were on, the new origin, and the full link, and you can decide -to go just to the new origin, or go to the full URL, after reviewing it for -tracking data. - -Also, you may click links that store your trust of that relationship with -various expiration times, the same way you would trust geolocation requests -for a particular origin for a period of time. The database used is actually -the new origin's cookie file. Since the origin prompt is loaded in the new -origin's context, I can set a cookie on behalf of the new origin. The -expiration time of the trust is the expiration time of the cookie. The -cookie implementation in webkit automatically expires the trust as part of -how cookies work. Each time you cross an origin, the origin crossing page -checks the cookie to see if trust is still established. If so, it will use -window.location.replace() to continue on automatically. The initial page -renders blank until the trust is invalidated, in which case the content of -the gate is made visible. - -However, the new origin is technically able to mess with those cookies, so -a website could set trust for an origin crossing. I have addressed that by -hashing the key with a salt, and setting the real expiration time as the -value, along with an HMAC to verify the contents of the value. If the -cookie is messed with in any way, the trust will be disabled, and the -prompt will appear again. So it has a fail-safe function. - -I know it seems a bit convoluted, but it just started out as a nice little -rabbit hole, and I just wanted to get something workable. At first I -thought using the cookie expiration time was convenient, but then when I -realized that I needed to protect the cookie, things got a bit hairy. But -it works. - -Each profile is, by default, stored in ~/.surf/origins/$origin/ -The interesting side effect is that if there is a problem where a website -relies on the cross-site cookie vulnerability to make a connection, you can -simply make a symbolic link from one origin folder to another, and they -will share the same profile. And if you want to delete cookies and/or cache -for a particular origin, you just rm -rf the origin's profile folder, and -don't have to interfere with your other sites that are working just fine. - -One thing I don't handle are cross-origins POSTs. They just end up as GET -requests right now. I intend to do something about that, but I haven't -figured that out yet. - -I have only been using this functionality for a few days myself, so I have -absolutely no feedback yet. I wanted to provide the first implementation of -the management of identity as a system resource the same way that things -like geolocation, camera, and microphone resources are managed in browsers -and mobile apps. - -Currently, Mozilla and Tor have are working on third-party tracking issues -in Firefox. -https://blog.mozilla.org/privacy/2014/11/10/introducing-polaris-privacy-initiative-to-accelerate-user-focused-privacy-online/ - -Up to this point, Tor has provided a patch that double-keys cookies with -the origin domain, but no other progress is visible. I have seen no -discussion of how horizontal isolation is supposed to happen, and I wanted -to show people that it can be done, and this is one way it can be done, and -to compel the other browser makers to catch up, and hopefully the community -can work toward a standard *without* the tracking loopholes, by showing -people what a *complete* solution looks like. - -Thank you, - -Ben Woolley - -Patch: http://lists.suckless.org/dev/att-25070/0005-same-origin-policy.patch From ff9efe22ae24dc215ca896ded69fd8939582f990 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 23 Jul 2017 17:17:03 -0400 Subject: [PATCH 295/337] Fix unused imports and removeRow override. Override removeRows instead of removeRow. > removeRow is not virtual in C++, so if this gets called by Qt > internally for some reason, it wouldn't use the overloaded version - > so I think it'd be better to implement removeRows and then use > removeRow without overloading that - The-Compiler --- qutebrowser/completion/models/histcategory.py | 6 +++--- qutebrowser/completion/models/listcategory.py | 3 +-- tests/unit/completion/test_histcategory.py | 6 ++---- tests/unit/completion/test_listcategory.py | 3 --- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index 56b7c3819..6fdab0cdb 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -25,7 +25,6 @@ from PyQt5.QtSql import QSqlQueryModel from qutebrowser.misc import sql from qutebrowser.utils import debug -from qutebrowser.commands import cmdexc from qutebrowser.config import config @@ -90,9 +89,10 @@ class HistoryCategory(QSqlQueryModel): self._query.run(pat=pattern) self.setQuery(self._query) - def removeRow(self, _row, _col): - """Re-run sql query to respond to a row removal.""" + def removeRows(self, _row, _count, _parent=None): + """Override QAbstractItemModel::removeRows to re-run sql query.""" # re-run query to reload updated table with debug.log_time('sql', 'Re-running completion query post-delete'): self._query.run() self.setQuery(self._query) + return True diff --git a/qutebrowser/completion/models/listcategory.py b/qutebrowser/completion/models/listcategory.py index 5f537d4d7..b1ad77bae 100644 --- a/qutebrowser/completion/models/listcategory.py +++ b/qutebrowser/completion/models/listcategory.py @@ -21,11 +21,10 @@ import re -from PyQt5.QtCore import Qt, QSortFilterProxyModel, QModelIndex, QRegExp +from PyQt5.QtCore import Qt, QSortFilterProxyModel, QRegExp from PyQt5.QtGui import QStandardItem, QStandardItemModel from qutebrowser.utils import qtutils -from qutebrowser.commands import cmdexc class ListCategory(QSortFilterProxyModel): diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index 7044f1281..a0b2b0144 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -19,7 +19,6 @@ """Test the web history completion category.""" -import unittest.mock import datetime import pytest @@ -27,7 +26,6 @@ from PyQt5.QtCore import QModelIndex from qutebrowser.misc import sql from qutebrowser.completion.models import histcategory -from qutebrowser.commands import cmdexc @pytest.fixture @@ -141,7 +139,7 @@ def test_sorting(max_items, before, after, model_validator, hist, config_stub): model_validator.validate(after) -def test_remove_row(hist, model_validator): +def test_remove_rows(hist, model_validator): hist.insert({'url': 'foo', 'title': 'Foo'}) hist.insert({'url': 'bar', 'title': 'Bar'}) cat = histcategory.HistoryCategory() @@ -149,5 +147,5 @@ def test_remove_row(hist, model_validator): cat.set_pattern('') hist.delete('url', 'foo') # histcategory does not care which index was removed, it just regenerates - cat.removeRow(QModelIndex(), QModelIndex()) + cat.removeRows(QModelIndex(), 1) model_validator.validate([('bar', 'Bar', '')]) diff --git a/tests/unit/completion/test_listcategory.py b/tests/unit/completion/test_listcategory.py index df1d1de5f..8d8936167 100644 --- a/tests/unit/completion/test_listcategory.py +++ b/tests/unit/completion/test_listcategory.py @@ -19,12 +19,9 @@ """Tests for CompletionFilterModel.""" -from unittest import mock - import pytest from qutebrowser.completion.models import listcategory -from qutebrowser.commands import cmdexc @pytest.mark.parametrize('pattern, before, after', [ From 2ad4cdd729d1e495c023168e05cbb99cc1181f45 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 23 Jul 2017 18:09:10 -0400 Subject: [PATCH 296/337] Fix web-history-max-items-crash. Fixes #2849, where pressing 'o' with web-history-max-items set and no history items would cause a crash as the query result is empty. --- qutebrowser/completion/models/histcategory.py | 4 ++++ tests/unit/completion/test_histcategory.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index 612eb0bf4..2977bfee4 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -72,6 +72,10 @@ class HistoryCategory(QSqlQueryModel): 'ORDER BY last_atime DESC LIMIT :limit)', ])).run(limit=max_items).value() + if not min_atime: + # if there are no history items, min_atime may be '' (issue #2849) + return '' + return "AND last_atime >= {}".format(min_atime) def set_pattern(self, pattern): diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index c53c86ee1..e4b06eaf6 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -126,7 +126,8 @@ def test_set_pattern(pattern, before, after, model_validator, hist): ], [ ('b', 'b', '2017-06-16'), ('c', 'c', '2017-05-16'), - ]) + ]), + (1, [], []), # issue 2849 (crash with empty history) ]) def test_sorting(max_items, before, after, model_validator, hist, config_stub): """Validate the filtering and sorting results of set_pattern.""" From 07b2fde2de912ac43803aa8eb4c7a6f671195894 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 24 Jul 2017 08:05:51 +0200 Subject: [PATCH 297/337] Mark test_version as flaky Sometimes it fails on Travis with empty output for no apparent reason --- tests/end2end/test_invocations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end2end/test_invocations.py b/tests/end2end/test_invocations.py index ed622b4c3..d1f4d0e23 100644 --- a/tests/end2end/test_invocations.py +++ b/tests/end2end/test_invocations.py @@ -163,6 +163,7 @@ def test_optimize(request, quteproc_new, capfd, level): @pytest.mark.not_frozen +@pytest.mark.flaky # Fails sometimes with empty output... def test_version(request): """Test invocation with --version argument.""" args = ['-m', 'qutebrowser', '--version'] + _base_args(request.config) From 99559b24e39c618c6f2e5fa0d60e58634462814b Mon Sep 17 00:00:00 2001 From: Penaz Date: Mon, 24 Jul 2017 10:58:45 +0200 Subject: [PATCH 298/337] Update FAQ.asciidoc --- FAQ.asciidoc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/FAQ.asciidoc b/FAQ.asciidoc index 9a540160a..38a159473 100644 --- a/FAQ.asciidoc +++ b/FAQ.asciidoc @@ -225,6 +225,18 @@ it's still https://bugreports.qt.io/browse/QTBUG-42417?jql=component%20%3D%20WebKit%20and%20resolution%20%3D%20Done%20and%20fixVersion%20in%20(5.4.0%2C%20%225.4.0%20Alpha%22%2C%20%225.4.0%20Beta%22%2C%20%225.4.0%20RC%22)%20and%20priority%20in%20(%22P2%3A%20Important%22%2C%20%22P1%3A%20Critical%22%2C%20%22P0%3A%20Blocker%22)[nearly 20 important bugs]. +When using QtWebEngine, qutebrowser reports "Render Process Crashed" and the console prints a traceback on Gentoo Linux or another Source-Based Distro:: + As stated in https://gcc.gnu.org/gcc-6/changes.html[GCC's Website] GCC 6 has introduced some optimizations that could break non-conforming codebases, like QtWebEngine. + + As a workaround, you can disable the nullpointer check optimization by adding the -fno-delete-null-pointer-checks flag while compiling. + + On gentoo, you just need to add it into your make.conf, like this: + + + CFLAGS="... -fno-delete-null-pointer-checks" + CXXFLAGS="... -fno-delete-null-pointer-checks" ++ +And then re-emerging qtwebengine with: + + + emerge -1 qtwebengine + My issue is not listed.:: If you experience any segfaults or crashes, you can report the issue in https://github.com/qutebrowser/qutebrowser/issues[the issue tracker] or From f09423efe5871e48c70aede659395743f668672a Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 24 Jul 2017 08:14:34 -0400 Subject: [PATCH 299/337] Abort resizeEvent if model is None. Some reports came in that a resizeEvent was causing a crash due to the model being none in the CompletionView. Fixes #2854. --- qutebrowser/completion/completionwidget.py | 2 ++ tests/unit/completion/test_completionwidget.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 65f1de76c..4674beb69 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -148,6 +148,8 @@ class CompletionView(QTreeView): def _resize_columns(self): """Resize the completion columns based on column_widths.""" + if self.model() is None: + return width = self.size().width() column_widths = self.model().column_widths pixel_widths = [(width * perc // 100) for perc in column_widths] diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 22d150fd8..2d4c8e744 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -240,3 +240,8 @@ def test_completion_item_del_no_selection(completionview): with pytest.raises(cmdexc.CommandError, match='No item selected!'): completionview.completion_item_del() assert not func.called + + +def test_resize_no_model(completionview, qtbot): + """Ensure no crash if resizeEvent is triggered with no model (#2854).""" + completionview.resizeEvent(None) From c1b883083168a470ae0c22a1d98016c648f5465c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Jul 2017 16:04:11 +0200 Subject: [PATCH 300/337] Update urllib3 from 1.21.1 to 1.22 --- misc/requirements/requirements-codecov.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-codecov.txt b/misc/requirements/requirements-codecov.txt index 86f78562d..c03e32477 100644 --- a/misc/requirements/requirements-codecov.txt +++ b/misc/requirements/requirements-codecov.txt @@ -6,4 +6,4 @@ codecov==2.0.9 coverage==4.4.1 idna==2.5 requests==2.18.1 -urllib3==1.21.1 +urllib3==1.22 From 920fae02c1962fbd7c9adf6af8e7047bf3731866 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Jul 2017 16:04:13 +0200 Subject: [PATCH 301/337] Update urllib3 from 1.21.1 to 1.22 --- misc/requirements/requirements-pylint-master.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pylint-master.txt b/misc/requirements/requirements-pylint-master.txt index 37e705b7a..7ccb7786c 100644 --- a/misc/requirements/requirements-pylint-master.txt +++ b/misc/requirements/requirements-pylint-master.txt @@ -14,5 +14,5 @@ requests==2.18.1 six==1.10.0 uritemplate==3.0.0 uritemplate.py==3.0.2 -urllib3==1.21.1 +urllib3==1.22 wrapt==1.10.10 From c49e5f84d92f452d148aa5c70715e3143a2f5546 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Jul 2017 16:04:14 +0200 Subject: [PATCH 302/337] Update urllib3 from 1.21.1 to 1.22 --- misc/requirements/requirements-pylint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pylint.txt b/misc/requirements/requirements-pylint.txt index a76d0dbf4..875947f87 100644 --- a/misc/requirements/requirements-pylint.txt +++ b/misc/requirements/requirements-pylint.txt @@ -14,5 +14,5 @@ requests==2.18.1 six==1.10.0 uritemplate==3.0.0 uritemplate.py==3.0.2 -urllib3==1.21.1 +urllib3==1.22 wrapt==1.10.10 From 96ed6668e5378f954818947434bc7920f7021e46 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Jul 2017 16:04:15 +0200 Subject: [PATCH 303/337] Update setuptools from 36.2.0 to 36.2.1 --- misc/requirements/requirements-pip.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pip.txt b/misc/requirements/requirements-pip.txt index 55c1f873e..923a7ef03 100644 --- a/misc/requirements/requirements-pip.txt +++ b/misc/requirements/requirements-pip.txt @@ -3,6 +3,6 @@ appdirs==1.4.3 packaging==16.8 pyparsing==2.2.0 -setuptools==36.2.0 +setuptools==36.2.1 six==1.10.0 wheel==0.29.0 From f8312e9502897f4ec5945315f05b44e3e55c554d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Jul 2017 16:04:17 +0200 Subject: [PATCH 304/337] Update decorator from 4.1.1 to 4.1.2 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 91cecb798..c18a1a87c 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -5,7 +5,7 @@ cheroot==5.7.0 click==6.7 # colorama==0.3.9 coverage==4.4.1 -decorator==4.1.1 +decorator==4.1.2 EasyProcess==0.2.3 fields==5.0.0 Flask==0.12.2 From b70f56e87ff3e0168b74c9c6261a7d42063c3833 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Jul 2017 16:04:18 +0200 Subject: [PATCH 305/337] Update hypothesis from 3.13.0 to 3.14.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index c18a1a87c..fd8ae0560 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -12,7 +12,7 @@ Flask==0.12.2 glob2==0.5 httpbin==0.5.0 hunter==1.4.1 -hypothesis==3.13.0 +hypothesis==3.14.0 itsdangerous==0.24 # Jinja2==2.9.6 Mako==1.0.7 From 734acd628eec681c0bda17aed556c371fe0f4150 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Jul 2017 16:04:20 +0200 Subject: [PATCH 306/337] Update pytest-benchmark from 3.0.0 to 3.1.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index fd8ae0560..1b92b9efe 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -22,7 +22,7 @@ parse-type==0.3.4 py==1.4.34 pytest==3.1.3 pytest-bdd==2.18.2 -pytest-benchmark==3.0.0 +pytest-benchmark==3.1.0 pytest-catchlog==1.2.2 pytest-cov==2.5.1 pytest-faulthandler==1.3.1 From 38664f9a0aeca98f73a1189471ab9e6da1109754 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Jul 2017 16:04:21 +0200 Subject: [PATCH 307/337] Update pytest-mock from 1.6.0 to 1.6.2 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 1b92b9efe..5007e6993 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -27,7 +27,7 @@ pytest-catchlog==1.2.2 pytest-cov==2.5.1 pytest-faulthandler==1.3.1 pytest-instafail==0.3.0 -pytest-mock==1.6.0 +pytest-mock==1.6.2 pytest-qt==2.1.2 pytest-repeat==0.4.1 pytest-rerunfailures==2.2 From c4d7cc79b5825fe86c389e0f5d6ced220f047b42 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Jul 2017 16:04:22 +0200 Subject: [PATCH 308/337] Update vulture from 0.16 to 0.19 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 5007e6993..1e8a25f76 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -35,5 +35,5 @@ pytest-travis-fold==1.2.0 pytest-xvfb==1.0.0 PyVirtualDisplay==0.2.1 six==1.10.0 -vulture==0.16 +vulture==0.19 Werkzeug==0.12.2 From 05dba381901ec96e5819b42c5bb9ef9c52e58770 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 24 Jul 2017 16:04:24 +0200 Subject: [PATCH 309/337] Update vulture from 0.16 to 0.19 --- misc/requirements/requirements-vulture.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-vulture.txt b/misc/requirements/requirements-vulture.txt index f2ba7dcbf..e9333c8fb 100644 --- a/misc/requirements/requirements-vulture.txt +++ b/misc/requirements/requirements-vulture.txt @@ -1,3 +1,3 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -vulture==0.16 +vulture==0.19 From 79c088d3a467d6cc605f811af3c8a9847a4c3ce5 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 24 Jul 2017 16:52:36 +0200 Subject: [PATCH 310/337] pytest.ini: Add benchmark-columns --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 225d64d9a..36ff6b870 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = --strict -rfEw --faulthandler-timeout=70 --instafail --pythonwarnings error +addopts = --strict -rfEw --faulthandler-timeout=70 --instafail --pythonwarnings error --benchmark-columns=Min,Max,Mean testpaths = tests markers = gui: Tests using the GUI (e.g. spawning widgets) From 3de0b1507343aef10b99e43d9d53d7fa3f9ffe73 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 25 Jul 2017 16:00:52 +0200 Subject: [PATCH 311/337] Delay showing the "renderer process killed" error page a bit Sometimes, we get another error with "Renderer process was killed" and the data: URL for the error page. This is probably because the renderer process wasn't restarted yet. This hopefully helps. --- qutebrowser/mainwindow/tabbedbrowser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index d74f74b4a..e5ec5ac4a 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -713,7 +713,7 @@ class TabbedBrowser(tabwidget.TabWidget): error_page = jinja.render( 'error.html', title="Error loading {}".format(url_string), url=url_string, error=msg, icon='') - QTimer.singleShot(0, lambda: tab.set_html(error_page)) + QTimer.singleShot(100, lambda: tab.set_html(error_page)) log.webview.error(msg) else: # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-58698 From 792a01ba6da59aa802f13f833fb77234e0d70615 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 25 Jul 2017 16:56:38 +0200 Subject: [PATCH 312/337] Try to stabilize renderer process test --- qutebrowser/mainwindow/tabbedbrowser.py | 6 +++++- tests/end2end/features/misc.feature | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index e5ec5ac4a..dd66ad8a2 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -708,12 +708,16 @@ class TabbedBrowser(tabwidget.TabWidget): } msg = messages[status] + def show_error_page(html): + tab.set_html(html) + log.webview.debug("Showed error page for renderer termination") + if qtutils.version_check('5.9'): url_string = tab.url(requested=True).toDisplayString() error_page = jinja.render( 'error.html', title="Error loading {}".format(url_string), url=url_string, error=msg, icon='') - QTimer.singleShot(100, lambda: tab.set_html(error_page)) + QTimer.singleShot(100, lambda: show_error_page(error_page)) log.webview.error(msg) else: # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-58698 diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index bf2b05697..2c4370576 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -700,6 +700,7 @@ Feature: Various utility commands. And I open data/numbers/2.txt in a new tab And I run :open chrome://kill And I wait for "Renderer process was killed" in the log + And I wait for "Showed error page for renderer termination" in the log And I open data/numbers/3.txt Then no crash should happen From 5ecda25fdb0d0231c40b5153a0c67eab3474303c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 25 Jul 2017 17:35:42 +0200 Subject: [PATCH 313/337] Fix renderer process test for older Qt versions --- qutebrowser/mainwindow/tabbedbrowser.py | 3 +-- tests/end2end/features/misc.feature | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index dd66ad8a2..50758a025 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -710,7 +710,7 @@ class TabbedBrowser(tabwidget.TabWidget): def show_error_page(html): tab.set_html(html) - log.webview.debug("Showed error page for renderer termination") + log.webview.error(msg) if qtutils.version_check('5.9'): url_string = tab.url(requested=True).toDisplayString() @@ -718,7 +718,6 @@ class TabbedBrowser(tabwidget.TabWidget): 'error.html', title="Error loading {}".format(url_string), url=url_string, error=msg, icon='') QTimer.singleShot(100, lambda: show_error_page(error_page)) - log.webview.error(msg) else: # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-58698 message.error(msg) diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index 2c4370576..bf2b05697 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -700,7 +700,6 @@ Feature: Various utility commands. And I open data/numbers/2.txt in a new tab And I run :open chrome://kill And I wait for "Renderer process was killed" in the log - And I wait for "Showed error page for renderer termination" in the log And I open data/numbers/3.txt Then no crash should happen From 79e7eb64951ac8d9c8566464d4f414e4f965876c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 25 Jul 2017 17:37:18 +0200 Subject: [PATCH 314/337] pytest: Show Median instead of Mean for benchmarks --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 36ff6b870..08273ef8d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = --strict -rfEw --faulthandler-timeout=70 --instafail --pythonwarnings error --benchmark-columns=Min,Max,Mean +addopts = --strict -rfEw --faulthandler-timeout=70 --instafail --pythonwarnings error --benchmark-columns=Min,Max,Median testpaths = tests markers = gui: Tests using the GUI (e.g. spawning widgets) From 1929883485c3368cb4ca7b9dc6eb97872a01c9d2 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 25 Jul 2017 12:55:44 -0400 Subject: [PATCH 315/337] Fix bind completion for bindings with arguments. When a key is bound to a command line that includes one or more arguments to a command, bind completion should show the whole command for the "Current" category, and use only the command name to look up the description. Fixes #2859, where a crash was caused by looking up the description by the full command text rather than just the name. --- qutebrowser/completion/models/miscmodels.py | 7 ++++--- tests/unit/completion/test_models.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 167eccde8..21a730a75 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -126,11 +126,12 @@ def bind(key): key: the key being bound. """ model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) - cmd_name = objreg.get('key-config').get_bindings_for('normal').get(key) + cmd_text = objreg.get('key-config').get_bindings_for('normal').get(key) - if cmd_name: + if cmd_text: + cmd_name = cmd_text.split(' ')[0] cmd = cmdutils.cmd_dict.get(cmd_name) - data = [(cmd_name, cmd.desc, key)] + data = [(cmd_text, cmd.desc, key)] model.add_category(listcategory.ListCategory("Current", data)) cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 83c3d3e7d..6907dff5d 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -583,7 +583,7 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, _patch_cmdutils(monkeypatch, stubs, 'qutebrowser.completion.models.miscmodels.cmdutils') config_stub.data['aliases'] = {'rock': 'roll'} - key_config_stub.set_bindings_for('normal', {'s': 'stop', + key_config_stub.set_bindings_for('normal', {'s': 'stop now', 'rr': 'roll', 'ro': 'rock'}) model = miscmodels.bind('s') @@ -593,14 +593,14 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub, _check_completions(model, { "Current": [ - ('stop', 'stop qutebrowser', 's'), + ('stop now', 'stop qutebrowser', 's'), ], "Commands": [ ('drop', 'drop all user data', ''), ('hide', '', ''), ('rock', "Alias for 'roll'", 'ro'), ('roll', 'never gonna give you up', 'rr'), - ('stop', 'stop qutebrowser', 's'), + ('stop', 'stop qutebrowser', ''), ] }) From 32fa1ff1e990e08110dfc7fe41b57d82bbdf2c6d Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 23 Jul 2017 19:32:24 -0400 Subject: [PATCH 316/337] Expand history completion results if on last index. When tabbing to the last index of history completion, call expandAll which will call fetchMore to retrieve more query results, if available. Calling fetchMore directly will not update the view, and for some reason self.expand(idx.parent()) and self.expand(self.model().index(idx.row(), 0)) did not work, so I'm using expandAll. Fixes #2841. --- qutebrowser/completion/completionwidget.py | 4 +++ .../unit/completion/test_completionwidget.py | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 6e1e51680..f6c72027b 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -255,6 +255,10 @@ class CompletionView(QTreeView): selmodel.setCurrentIndex( idx, QItemSelectionModel.ClearAndSelect | QItemSelectionModel.Rows) + # if the last item is focused, try to fetch more + if idx.row() == self.model().rowCount(idx.parent()) - 1: + self.expandAll() + count = self.model().count() if count == 0: self.hide() diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index 207e557a8..09552bcda 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -65,6 +65,9 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry, } # mock the Completer that the widget creates in its constructor mocker.patch('qutebrowser.completion.completer.Completer', autospec=True) + mocker.patch( + 'qutebrowser.completion.completiondelegate.CompletionItemDelegate', + new=lambda *_: None) view = completionwidget.CompletionView(win_id=0) qtbot.addWidget(view) return view @@ -185,6 +188,37 @@ def test_completion_item_focus_no_model(which, completionview, qtbot): completionview.completion_item_focus(which) +def test_completion_item_focus_fetch(completionview, qtbot): + """Test that on_next_prev_item moves the selection properly. + + Args: + which: the direction in which to move the selection. + tree: Each list represents a completion category, with each string + being an item under that category. + expected: expected argument from on_selection_changed for each + successive movement. None implies no signal should be + emitted. + """ + model = completionmodel.CompletionModel() + cat = mock.Mock(spec=['layoutChanged', 'layoutAboutToBeChanged', + 'canFetchMore', 'fetchMore', 'rowCount', 'index', 'data']) + cat.canFetchMore = lambda *_: True + cat.rowCount = lambda *_: 2 + cat.fetchMore = mock.Mock() + model.add_category(cat) + completionview.set_model(model) + # clear the fetchMore call that happens on set_model + cat.reset_mock() + + # not at end, fetchMore shouldn't be called + completionview.completion_item_focus('next') + assert not cat.fetchMore.called + + # at end, fetchMore should be called + completionview.completion_item_focus('next') + assert cat.fetchMore.called + + @pytest.mark.parametrize('show', ['always', 'auto', 'never']) @pytest.mark.parametrize('rows', [[], ['Aa'], ['Aa', 'Bb']]) @pytest.mark.parametrize('quick_complete', [True, False]) From a942613d7fea62932de3ced4008ebbb8ae190bc4 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 27 Jul 2017 09:22:12 +0200 Subject: [PATCH 317/337] Use ctypes instead of PyOpenGL for QtWebEngine Nvidia workaround Fixes #2821 --- CHANGELOG.asciidoc | 2 ++ README.asciidoc | 1 - qutebrowser/browser/webengine/webenginesettings.py | 8 ++++++++ qutebrowser/misc/earlyinit.py | 6 ------ qutebrowser/utils/version.py | 1 - requirements.txt | 1 - tests/unit/utils/test_version.py | 1 - 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d28efa7e6..4591627c6 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -28,6 +28,8 @@ Breaking changes - New dependency on ruamel.yaml; dropped PyYAML dependency. - The QtWebEngine backend is now used by default if available. - New config system which ignores the old config file. +- The depedency on PyOpenGL (when using QtWebEngine) got removed. Note + that PyQt5.QtOpenGL is still a depdency. Major changes ~~~~~~~~~~~~~ diff --git a/README.asciidoc b/README.asciidoc index f7ad0f64d..bf6225062 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -118,7 +118,6 @@ The following software and libraries are required to run qutebrowser: * http://jinja.pocoo.org/[jinja2] * http://pygments.org/[pygments] * http://pyyaml.org/wiki/PyYAML[PyYAML] -* http://pyopengl.sourceforge.net/[PyOpenGL] when using QtWebEngine The following libraries are optional: diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 27ef60cb6..1148dbad9 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -28,6 +28,9 @@ Module attributes: """ import os +import sys +import ctypes +import ctypes.util from PyQt5.QtGui import QFont from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile, @@ -199,6 +202,11 @@ def init(args): if args.enable_webengine_inspector: os.environ['QTWEBENGINE_REMOTE_DEBUGGING'] = str(utils.random_port()) + # WORKAROUND for + # https://bugs.launchpad.net/ubuntu/+source/python-qt4/+bug/941826 + if sys.platform == 'linux': + ctypes.CDLL(ctypes.util.find_library("GL"), mode=ctypes.RTLD_GLOBAL) + _init_profiles() # We need to do this here as a WORKAROUND for diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index e43cd8891..df50b8d08 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -344,12 +344,6 @@ def check_libraries(backend): modules['PyQt5.QtWebEngineWidgets'] = _missing_str("QtWebEngine", webengine=True) modules['PyQt5.QtOpenGL'] = _missing_str("PyQt5.QtOpenGL") - # Workaround for a black screen with some setups - # https://github.com/spyder-ide/spyder/issues/3226 - if not os.environ.get('QUTE_NO_OPENGL_WORKAROUND'): - # Hide "No OpenGL_accelerate module loaded: ..." message - logging.getLogger('OpenGL.acceleratesupport').propagate = False - modules['OpenGL.GL'] = _missing_str("PyOpenGL") else: assert backend == 'webkit' modules['PyQt5.QtWebKit'] = _missing_str("PyQt5.QtWebKit") diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index fe3e1aedb..0b650a97e 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -186,7 +186,6 @@ def _module_versions(): ('yaml', ['__version__']), ('cssutils', ['__version__']), ('typing', []), - ('OpenGL', ['__version__']), ('PyQt5.QtWebEngineWidgets', []), ('PyQt5.QtWebKitWidgets', []), ]) diff --git a/requirements.txt b/requirements.txt index cbf9ba407..b2cc93c1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,3 @@ MarkupSafe==1.0 Pygments==2.2.0 pyPEG2==2.15.2 PyYAML==3.12 -PyOpenGL==3.1.0 diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 339e70987..f9da37841 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -495,7 +495,6 @@ class ImportFake: ('yaml', True), ('cssutils', True), ('typing', True), - ('OpenGL', True), ('PyQt5.QtWebEngineWidgets', True), ('PyQt5.QtWebKitWidgets', True), ]) From e4f776448e2208615b7d8188b2aaaf0ecb6f8c26 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 27 Jul 2017 09:25:33 +0200 Subject: [PATCH 318/337] Fix typo --- CHANGELOG.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 4591627c6..9f93d26bd 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -29,7 +29,7 @@ Breaking changes - The QtWebEngine backend is now used by default if available. - New config system which ignores the old config file. - The depedency on PyOpenGL (when using QtWebEngine) got removed. Note - that PyQt5.QtOpenGL is still a depdency. + that PyQt5.QtOpenGL is still a dependency. Major changes ~~~~~~~~~~~~~ From 629f6a6876467746e05bf133ae451751936eb835 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 27 Jul 2017 09:56:34 +0200 Subject: [PATCH 319/337] Remove unused import --- qutebrowser/misc/earlyinit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index df50b8d08..a64a2799b 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -36,7 +36,6 @@ import traceback import signal import importlib import datetime -import logging try: import tkinter except ImportError: From 2b07b3db2b2084237ab4f0cb14623bfc0358a3c3 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 27 Jul 2017 12:18:59 +0200 Subject: [PATCH 320/337] Update 'not code' label in CONTRIBUTING --- CONTRIBUTING.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.asciidoc b/CONTRIBUTING.asciidoc index fc039ad03..7d42cad28 100644 --- a/CONTRIBUTING.asciidoc +++ b/CONTRIBUTING.asciidoc @@ -45,8 +45,8 @@ pointers: * https://github.com/qutebrowser/qutebrowser/labels/easy[Issues which should be easy to solve] -* https://github.com/qutebrowser/qutebrowser/labels/not%20code[Issues which -require little/no coding] +* https://github.com/qutebrowser/qutebrowser/labels/component%3A%20docs[Documentation +* issues which require little/no coding] If you prefer C++ or Javascript to Python, see the relevant issues which involve work in those languages: From c6cb6ccd077fd98bb07c3fd385048e607398a0c1 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 28 Jul 2017 08:14:38 -0400 Subject: [PATCH 321/337] Fix fetch/delete sql category bug. Fixes #2868, where pressing then in history completion (with > 256 items) would cause later items to disappear (and cause a crash if you try to delete again). Cause: Scrolling to the bottom would fetch an additional 256 items (in addition to the 256 that are fetched at first). Deleting causes the query to re-run, but it only fetches the initial 256 items, so the current index is now invalid. Fix: After deleting from the history category, call fetchMore until it has enough rows populated that the current index is valid. --- qutebrowser/completion/models/histcategory.py | 4 +++- tests/unit/completion/test_histcategory.py | 21 ++++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index 42301c0dd..606351440 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -93,10 +93,12 @@ class HistoryCategory(QSqlQueryModel): self._query.run(pat=pattern) self.setQuery(self._query) - def removeRows(self, _row, _count, _parent=None): + def removeRows(self, row, _count, _parent=None): """Override QAbstractItemModel::removeRows to re-run sql query.""" # re-run query to reload updated table with debug.log_time('sql', 'Re-running completion query post-delete'): self._query.run() self.setQuery(self._query) + while self.rowCount() < row: + self.fetchMore() return True diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index db4071502..1397b8b5f 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -22,7 +22,6 @@ import datetime import pytest -from PyQt5.QtCore import QModelIndex from qutebrowser.misc import sql from qutebrowser.completion.models import histcategory @@ -147,6 +146,22 @@ def test_remove_rows(hist, model_validator): model_validator.set_model(cat) cat.set_pattern('') hist.delete('url', 'foo') - # histcategory does not care which index was removed, it just regenerates - cat.removeRows(QModelIndex(), 1) + cat.removeRows(0, 1) model_validator.validate([('bar', 'Bar', '')]) + + +def test_remove_rows_fetch(hist): + """removeRows should fetch enough data to make the current index valid.""" + # we cannot use model_validator as it will fetch everything up front + hist.insert_batch({'url': [str(i) for i in range(300)]}) + cat = histcategory.HistoryCategory() + cat.set_pattern('') + + # sanity check that we didn't fetch everything up front + assert cat.rowCount() < 300 + cat.fetchMore() + assert cat.rowCount() == 300 + + hist.delete('url', '298') + cat.removeRows(297, 1) + assert cat.rowCount() == 299 From 1ab7bb83ccea803e51daf77e8e76bb58440390f2 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 28 Jul 2017 09:02:15 -0400 Subject: [PATCH 322/337] Support delete from :{quick,book}mark-load. Pressing ctrl-d in the completion menu for :quickmark-load/:bookmark-load will now delete the selected quickmark/bookmark. Resolves #2840. --- qutebrowser/completion/models/miscmodels.py | 20 +++++++++- tests/unit/completion/test_models.py | 42 +++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 21a730a75..a61b73ffa 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -60,17 +60,33 @@ def helptopic(): def quickmark(): """A CompletionModel filled with all quickmarks.""" + def delete(data): + """Delete a quickmark from the completion menu.""" + name = data[0] + quickmark_manager = objreg.get('quickmark-manager') + log.completion.debug('Deleting quickmark {}'.format(name)) + quickmark_manager.delete(name) + model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) marks = objreg.get('quickmark-manager').marks.items() - model.add_category(listcategory.ListCategory('Quickmarks', marks)) + model.add_category(listcategory.ListCategory('Quickmarks', marks, + delete_func=delete)) return model def bookmark(): """A CompletionModel filled with all bookmarks.""" + def delete(data): + """Delete a bookmark from the completion menu.""" + urlstr = data[0] + log.completion.debug('Deleting bookmark {}'.format(urlstr)) + bookmark_manager = objreg.get('bookmark-manager') + bookmark_manager.delete(urlstr) + model = completionmodel.CompletionModel(column_widths=(30, 70, 0)) marks = objreg.get('bookmark-manager').marks.items() - model.add_category(listcategory.ListCategory('Bookmarks', marks)) + model.add_category(listcategory.ListCategory('Bookmarks', marks, + delete_func=delete)) return model diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 6907dff5d..b4a940977 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -252,6 +252,27 @@ def test_quickmark_completion(qtmodeltester, quickmarks): }) +@pytest.mark.parametrize('row, removed', [ + (0, 'aw'), + (1, 'ddg'), + (2, 'wiki'), +]) +def test_quickmark_completion_delete(qtmodeltester, quickmarks, row, removed): + """Test deleting a quickmark from the quickmark completion model.""" + model = miscmodels.quickmark() + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + parent = model.index(0, 0) + idx = model.index(row, 0, parent) + + before = set(quickmarks.marks.keys()) + model.delete_cur_item(idx) + after = set(quickmarks.marks.keys()) + assert before.difference(after) == {removed} + + def test_bookmark_completion(qtmodeltester, bookmarks): """Test the results of bookmark completion.""" model = miscmodels.bookmark() @@ -268,6 +289,27 @@ def test_bookmark_completion(qtmodeltester, bookmarks): }) +@pytest.mark.parametrize('row, removed', [ + (0, 'http://qutebrowser.org'), + (1, 'https://github.com'), + (2, 'https://python.org'), +]) +def test_bookmark_completion_delete(qtmodeltester, bookmarks, row, removed): + """Test deleting a quickmark from the quickmark completion model.""" + model = miscmodels.bookmark() + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + parent = model.index(0, 0) + idx = model.index(row, 0, parent) + + before = set(bookmarks.marks.keys()) + model.delete_cur_item(idx) + after = set(bookmarks.marks.keys()) + assert before.difference(after) == {removed} + + def test_url_completion(qtmodeltester, web_history_populated, quickmarks, bookmarks): """Test the results of url completion. From 8e34b54cd7a55a4d0cae43d274a5f9ec9d895d82 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 29 Jul 2017 12:59:11 -0400 Subject: [PATCH 323/337] Remove obsolete signals. The added/removed signals for the urlmark managers are no longer used as the completion models are generated on-the-fly. The changed signal is still needed so the save-manager knows when to trigger a write to disk. Also removes session_manager.update_completion, which is no longer needed for the same reason as above. keyconf.changed cannot be removed, as it is still wired up to basekeyparser. Resolves #2874. --- qutebrowser/browser/urlmarks.py | 9 --------- qutebrowser/misc/sessions.py | 12 ++---------- tests/unit/misc/test_sessions.py | 13 ------------- 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index 013de408c..b7c93a994 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -77,13 +77,9 @@ class UrlMarkManager(QObject): Signals: changed: Emitted when anything changed. - added: Emitted when a new quickmark/bookmark was added. - removed: Emitted when an existing quickmark/bookmark was removed. """ changed = pyqtSignal() - added = pyqtSignal(str, str) - removed = pyqtSignal(str) def __init__(self, parent=None): """Initialize and read quickmarks.""" @@ -121,7 +117,6 @@ class UrlMarkManager(QObject): """ del self.marks[key] self.changed.emit() - self.removed.emit(key) class QuickmarkManager(UrlMarkManager): @@ -133,7 +128,6 @@ class QuickmarkManager(UrlMarkManager): - self.marks maps names to URLs. - changed gets emitted with the name as first argument and the URL as second argument. - - removed gets emitted with the name as argument. """ def _init_lineparser(self): @@ -193,7 +187,6 @@ class QuickmarkManager(UrlMarkManager): """Really set the quickmark.""" self.marks[name] = url self.changed.emit() - self.added.emit(name, url) log.misc.debug("Added quickmark {} for {}".format(name, url)) if name in self.marks: @@ -243,7 +236,6 @@ class BookmarkManager(UrlMarkManager): - self.marks maps URLs to titles. - changed gets emitted with the URL as first argument and the title as second argument. - - removed gets emitted with the URL as argument. """ def _init_lineparser(self): @@ -295,5 +287,4 @@ class BookmarkManager(UrlMarkManager): else: self.marks[urlstr] = title self.changed.emit() - self.added.emit(title, urlstr) return True diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 4fe0fe4c7..0a4fae5b7 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -23,7 +23,7 @@ import os import os.path import sip -from PyQt5.QtCore import pyqtSignal, QUrl, QObject, QPoint, QTimer +from PyQt5.QtCore import QUrl, QObject, QPoint, QTimer from PyQt5.QtWidgets import QApplication import yaml try: @@ -106,14 +106,8 @@ class SessionManager(QObject): closed. _current: The name of the currently loaded session, or None. did_load: Set when a session was loaded. - - Signals: - update_completion: Emitted when the session completion should get - updated. """ - update_completion = pyqtSignal() - def __init__(self, base_path, parent=None): super().__init__(parent) self._current = None @@ -303,8 +297,7 @@ class SessionManager(QObject): encoding='utf-8', allow_unicode=True) except (OSError, UnicodeEncodeError, yaml.YAMLError) as e: raise SessionError(e) - else: - self.update_completion.emit() + if load_next_time: state_config = objreg.get('state-config') state_config['general']['session'] = name @@ -425,7 +418,6 @@ class SessionManager(QObject): os.remove(path) except OSError as e: raise SessionError(e) - self.update_completion.emit() def list_sessions(self): """Get a list of all session names.""" diff --git a/tests/unit/misc/test_sessions.py b/tests/unit/misc/test_sessions.py index e77b6aa02..00311ec8e 100644 --- a/tests/unit/misc/test_sessions.py +++ b/tests/unit/misc/test_sessions.py @@ -215,11 +215,6 @@ class TestSave: objreg.delete('main-window', scope='window', window=0) objreg.delete('tabbed-browser', scope='window', window=0) - def test_update_completion_signal(self, sess_man, tmpdir, qtbot): - session_path = tmpdir / 'foo.yml' - with qtbot.waitSignal(sess_man.update_completion): - sess_man.save(str(session_path)) - def test_no_state_config(self, sess_man, tmpdir, state_config): session_path = tmpdir / 'foo.yml' sess_man.save(str(session_path)) @@ -367,14 +362,6 @@ class TestLoadTab: assert loaded_item.original_url == expected -def test_delete_update_completion_signal(sess_man, qtbot, tmpdir): - sess = tmpdir / 'foo.yml' - sess.ensure() - - with qtbot.waitSignal(sess_man.update_completion): - sess_man.delete(str(sess)) - - class TestListSessions: def test_no_sessions(self, tmpdir): From 57bf36156b281d4542308b391b3dd11fb62d6113 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:11 +0200 Subject: [PATCH 324/337] Update certifi from 2017.4.17 to 2017.7.27.1 --- misc/requirements/requirements-codecov.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-codecov.txt b/misc/requirements/requirements-codecov.txt index c03e32477..69b94ddf8 100644 --- a/misc/requirements/requirements-codecov.txt +++ b/misc/requirements/requirements-codecov.txt @@ -1,6 +1,6 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -certifi==2017.4.17 +certifi==2017.7.27.1 chardet==3.0.4 codecov==2.0.9 coverage==4.4.1 From 26b47bcb6e8a7fac7be4bda8c6c04f16e4d9fe88 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:13 +0200 Subject: [PATCH 325/337] Update certifi from 2017.4.17 to 2017.7.27.1 --- misc/requirements/requirements-pylint-master.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pylint-master.txt b/misc/requirements/requirements-pylint-master.txt index 7ccb7786c..84db8381e 100644 --- a/misc/requirements/requirements-pylint-master.txt +++ b/misc/requirements/requirements-pylint-master.txt @@ -1,7 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -e git+https://github.com/PyCQA/astroid.git#egg=astroid -certifi==2017.4.17 +certifi==2017.7.27.1 chardet==3.0.4 github3.py==0.9.6 idna==2.5 From 28c62a7f03969f9bae721188c7e528ed01d2dd47 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:14 +0200 Subject: [PATCH 326/337] Update certifi from 2017.4.17 to 2017.7.27.1 --- misc/requirements/requirements-pylint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pylint.txt b/misc/requirements/requirements-pylint.txt index 875947f87..3219113f3 100644 --- a/misc/requirements/requirements-pylint.txt +++ b/misc/requirements/requirements-pylint.txt @@ -1,7 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py astroid==1.5.3 -certifi==2017.4.17 +certifi==2017.7.27.1 chardet==3.0.4 github3.py==0.9.6 idna==2.5 From 67f3396ced3235bf8eacbd3ae9c552ff917b4229 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:15 +0200 Subject: [PATCH 327/337] Update requests from 2.18.1 to 2.18.2 --- misc/requirements/requirements-codecov.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-codecov.txt b/misc/requirements/requirements-codecov.txt index 69b94ddf8..9d6737a96 100644 --- a/misc/requirements/requirements-codecov.txt +++ b/misc/requirements/requirements-codecov.txt @@ -5,5 +5,5 @@ chardet==3.0.4 codecov==2.0.9 coverage==4.4.1 idna==2.5 -requests==2.18.1 +requests==2.18.2 urllib3==1.22 From 633026e8b3d0285896cf8fd422af319591c0ebe6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:17 +0200 Subject: [PATCH 328/337] Update requests from 2.18.1 to 2.18.2 --- misc/requirements/requirements-pylint-master.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pylint-master.txt b/misc/requirements/requirements-pylint-master.txt index 84db8381e..d4058b1d0 100644 --- a/misc/requirements/requirements-pylint-master.txt +++ b/misc/requirements/requirements-pylint-master.txt @@ -10,7 +10,7 @@ lazy-object-proxy==1.3.1 mccabe==0.6.1 -e git+https://github.com/PyCQA/pylint.git#egg=pylint ./scripts/dev/pylint_checkers -requests==2.18.1 +requests==2.18.2 six==1.10.0 uritemplate==3.0.0 uritemplate.py==3.0.2 From 92a520fa8ce2b7aa1feec082926090ceded86c4c Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:18 +0200 Subject: [PATCH 329/337] Update requests from 2.18.1 to 2.18.2 --- misc/requirements/requirements-pylint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pylint.txt b/misc/requirements/requirements-pylint.txt index 3219113f3..b5d44cb64 100644 --- a/misc/requirements/requirements-pylint.txt +++ b/misc/requirements/requirements-pylint.txt @@ -10,7 +10,7 @@ lazy-object-proxy==1.3.1 mccabe==0.6.1 pylint==1.7.2 ./scripts/dev/pylint_checkers -requests==2.18.1 +requests==2.18.2 six==1.10.0 uritemplate==3.0.0 uritemplate.py==3.0.2 From 2e4704aaa79cff33c5fe99be14216fe5ea05aa0d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:20 +0200 Subject: [PATCH 330/337] Update flake8-deprecated from 1.2 to 1.2.1 --- misc/requirements/requirements-flake8.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index f68bcb227..5e5980525 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -3,7 +3,7 @@ flake8==2.6.2 # rq.filter: < 3.0.0 flake8-copyright==0.2.0 flake8-debugger==1.4.0 # rq.filter: != 2.0.0 -flake8-deprecated==1.2 +flake8-deprecated==1.2.1 flake8-docstrings==1.0.3 # rq.filter: < 1.1.0 flake8-future-import==0.4.3 flake8-mock==0.3 From e4db036382878431cda559933c2c56fb8e92aaf4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:21 +0200 Subject: [PATCH 331/337] Update setuptools from 36.2.1 to 36.2.5 --- misc/requirements/requirements-pip.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pip.txt b/misc/requirements/requirements-pip.txt index 923a7ef03..3b36a0e5c 100644 --- a/misc/requirements/requirements-pip.txt +++ b/misc/requirements/requirements-pip.txt @@ -3,6 +3,6 @@ appdirs==1.4.3 packaging==16.8 pyparsing==2.2.0 -setuptools==36.2.1 +setuptools==36.2.5 six==1.10.0 wheel==0.29.0 From 240feaf547b0ab913ba26ed9250733568554f6eb Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:23 +0200 Subject: [PATCH 332/337] Update pytest-benchmark from 3.1.0 to 3.1.1 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 1e8a25f76..8e34197b1 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -22,7 +22,7 @@ parse-type==0.3.4 py==1.4.34 pytest==3.1.3 pytest-bdd==2.18.2 -pytest-benchmark==3.1.0 +pytest-benchmark==3.1.1 pytest-catchlog==1.2.2 pytest-cov==2.5.1 pytest-faulthandler==1.3.1 From 931d9cc372e0e4b7d5ee38da09a2a396386d9144 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:24 +0200 Subject: [PATCH 333/337] Update vulture from 0.19 to 0.21 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 8e34197b1..171013afb 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -35,5 +35,5 @@ pytest-travis-fold==1.2.0 pytest-xvfb==1.0.0 PyVirtualDisplay==0.2.1 six==1.10.0 -vulture==0.19 +vulture==0.21 Werkzeug==0.12.2 From dd252056237511ea6bee8f8b65e5c99f13d01b97 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 31 Jul 2017 16:07:25 +0200 Subject: [PATCH 334/337] Update vulture from 0.19 to 0.21 --- misc/requirements/requirements-vulture.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-vulture.txt b/misc/requirements/requirements-vulture.txt index e9333c8fb..d1c1bc41d 100644 --- a/misc/requirements/requirements-vulture.txt +++ b/misc/requirements/requirements-vulture.txt @@ -1,3 +1,3 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -vulture==0.19 +vulture==0.21 From bcba14a029f7725553e481c813591784ad5bbecf Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 31 Jul 2017 16:35:12 +0200 Subject: [PATCH 335/337] Adjust run_vulture.py for new vulture version --- scripts/dev/run_vulture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dev/run_vulture.py b/scripts/dev/run_vulture.py index 8a1886e11..db5ad8509 100755 --- a/scripts/dev/run_vulture.py +++ b/scripts/dev/run_vulture.py @@ -111,7 +111,7 @@ def filter_func(item): True if the missing function should be filtered/ignored, False otherwise. """ - return bool(re.match(r'[a-z]+[A-Z][a-zA-Z]+', str(item))) + return bool(re.match(r'[a-z]+[A-Z][a-zA-Z]+', item.name)) def report(items): @@ -125,7 +125,7 @@ def report(items): relpath = os.path.relpath(item.filename) path = relpath if not relpath.startswith('..') else item.filename output.append("{}:{}: Unused {} '{}'".format(path, item.lineno, - item.typ, item)) + item.typ, item.name)) return output From a329ce41b50fbfaca6b378a5c31e2255b956143a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 31 Jul 2017 16:41:43 +0200 Subject: [PATCH 336/337] Update vulture whitelist --- scripts/dev/run_vulture.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/dev/run_vulture.py b/scripts/dev/run_vulture.py index db5ad8509..2a0ecec44 100755 --- a/scripts/dev/run_vulture.py +++ b/scripts/dev/run_vulture.py @@ -89,6 +89,12 @@ def whitelist_generator(): # vulture doesn't notice the hasattr() and thus thinks netrc_used is unused # in NetworkManager.on_authentication_required yield 'PyQt5.QtNetwork.QNetworkReply.netrc_used' + yield 'qutebrowser.browser.downloads.last_used_directory' + yield 'PaintContext.clip' # from completiondelegate.py + yield 'logging.LogRecord.log_color' # from logging.py + yield 'scripts.utils.use_color' # from asciidoc2html.py + for attr in ['pyeval_output', 'log_clipboard', 'fake_clipboard']: + yield 'qutebrowser.misc.utilcmds.' + attr for attr in ['fileno', 'truncate', 'closed', 'readable']: yield 'qutebrowser.utils.qtutils.PyQIODevice.' + attr From 49b858e3599a361cf5c994358c6b189dddfb522c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 1 Aug 2017 16:00:12 +0200 Subject: [PATCH 337/337] Add more variants of fake apple URL to ignored ones --- qutebrowser/browser/history.py | 3 ++- tests/unit/browser/test_history.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index b9e791207..86e597bcd 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -185,7 +185,8 @@ class WebHistory(sql.SqlTable): # http://xn--pple-43d.com/ with # https://bugreports.qt.io/browse/QTBUG-60364 - if url in ['http://.com/', 'https://www..com/']: + if url in ['http://.com/', 'https://.com/', + 'http://www..com/', 'https://www..com/']: return None url = QUrl(url) diff --git a/tests/unit/browser/test_history.py b/tests/unit/browser/test_history.py index 81637f3d4..c109f44eb 100644 --- a/tests/unit/browser/test_history.py +++ b/tests/unit/browser/test_history.py @@ -281,6 +281,8 @@ def test_import_txt(hist, data_tmpdir, monkeypatch, stubs): # https://bugreports.qt.io/browse/QTBUG-60364 '12345 http://.com/', + '12345 https://.com/', + '12345 http://www..com/', '12345 https://www..com/', # issue #2646