Merge pull request #2295 from rcorre/really_complete

Completion refactor V3

.flake8

@@ -11,6 +11,7 @@ exclude = .*,__pycache__,resources.py
 #       (for pytest's __tracebackhide__)
 # F401: Unused import
 # N802: function name should be lowercase
+# N806: variable in function should be lowercase
 # P101: format string does contain unindexed parameters
 # P102: docstring does contain unindexed parameters
 # P103: other string does contain unindexed parameters
@@ -38,7 +39,7 @@ putty-ignore =
     /# pragma: no mccabe/ : +C901
     tests/*/test_*.py : +D100,D101,D401
     tests/conftest.py : +F403
-    tests/unit/browser/webkit/test_history.py : +N806
+    tests/unit/browser/test_history.py : +N806
     tests/helpers/fixtures.py : +N806
     tests/unit/browser/webkit/http/test_content_disposition.py : +D400
     scripts/dev/ci/appveyor_install.py : +FI53

INSTALL.asciidoc

@@ -27,7 +27,7 @@ Using the packages
 Install the dependencies via apt-get:
 
 ----
-# apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-jinja2 python3-pygments python3-yaml
+# apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-jinja2 python3-pygments python3-yaml python3-pyqt5.qtsql libqt5sql5-sqlite
 ----
 
 On Debian Stretch or Ubuntu 17.04 or later, it's also recommended to use the
@@ -53,7 +53,7 @@ Build it from git
 Install the dependencies via apt-get:
 
 ----
-# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python-tox python3-sip python3-dev
+# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python-tox python3-sip python3-dev python3-pyqt5.qtsql libqt5sql5-sqlite
 ----
 
 On Debian Stretch or Ubuntu 17.04 or later, it's also recommended to install

qutebrowser/app.py

@@ -42,9 +42,10 @@ except ImportError:
 
 import qutebrowser
 import qutebrowser.resources
-from qutebrowser.completion.models import instances as completionmodels
+from qutebrowser.completion.models import miscmodels
 from qutebrowser.commands import cmdutils, runners, cmdexc
 from qutebrowser.config import style, config, websettings, configexc
+from qutebrowser.config.parsers import keyconf
 from qutebrowser.browser import (urlmarks, adblock, history, browsertab,
                                  downloads)
 from qutebrowser.browser.network import proxy
@@ -53,10 +54,10 @@ from qutebrowser.browser.webkit.network import networkmanager
 from qutebrowser.keyinput import macros
 from qutebrowser.mainwindow import mainwindow, prompt
 from qutebrowser.misc import (readline, ipc, savemanager, sessions,
-                              crashsignal, earlyinit, objects)
+                              crashsignal, earlyinit, objects, sql)
 from qutebrowser.misc import utilcmds  # pylint: disable=unused-import
 from qutebrowser.utils import (log, version, message, utils, qtutils, urlutils,
-                               objreg, usertypes, standarddir, error, debug)
+                               objreg, usertypes, standarddir, error)
 # We import utilcmds to run the cmdutils.register decorators.
 
 
@@ -157,7 +158,7 @@ def init(args, crash_handler):
     QDesktopServices.setUrlHandler('https', open_desktopservices_url)
     QDesktopServices.setUrlHandler('qute', open_desktopservices_url)
 
-    QTimer.singleShot(10, functools.partial(_init_late_modules, args))
+    objreg.get('web-history').import_txt()
 
     log.init.debug("Init done!")
     crash_handler.raise_crashdlg()
@@ -421,6 +422,17 @@ def _init_modules(args, crash_handler):
     config.init(qApp)
     save_manager.init_autosave()
 
+    log.init.debug("Initializing keys...")
+    keyconf.init(qApp)
+
+    log.init.debug("Initializing sql...")
+    try:
+        sql.init(os.path.join(standarddir.data(), 'history.sqlite'))
+    except sql.SqlException as e:
+        error.handle_fatal_exc(e, args, 'Error initializing SQL',
+                               pre_text='Error initializing SQL')
+        sys.exit(usertypes.Exit.err_init)
+
     log.init.debug("Initializing web history...")
     history.init(qApp)
 
@@ -457,9 +469,6 @@ def _init_modules(args, crash_handler):
     diskcache = cache.DiskCache(standarddir.cache(), parent=qApp)
     objreg.register('cache', diskcache)
 
-    log.init.debug("Initializing completions...")
-    completionmodels.init()
-
     log.init.debug("Misc initialization...")
     if config.get('ui', 'hide-wayland-decoration'):
         os.environ['QT_WAYLAND_DISABLE_WINDOWDECORATION'] = '1'
@@ -470,23 +479,6 @@ def _init_modules(args, crash_handler):
     browsertab.init()
 
 
-def _init_late_modules(args):
-    """Initialize modules which can be inited after the window is shown."""
-    log.init.debug("Reading web history...")
-    reader = objreg.get('web-history').async_read()
-    with debug.log_time(log.init, 'Reading history'):
-        while True:
-            QApplication.processEvents()
-            try:
-                next(reader)
-            except StopIteration:
-                break
-            except (OSError, UnicodeDecodeError) as e:
-                error.handle_fatal_exc(e, args, "Error while initializing!",
-                                       pre_text="Error while initializing")
-                sys.exit(usertypes.Exit.err_init)
-
-
 class Quitter:
 
     """Utility class to quit/restart the QApplication.
@@ -751,7 +743,7 @@ class Quitter:
         QTimer.singleShot(0, functools.partial(qApp.exit, status))
 
     @cmdutils.register(instance='quitter', name='wq')
-    @cmdutils.argument('name', completion=usertypes.Completion.sessions)
+    @cmdutils.argument('name', completion=miscmodels.session)
     def save_and_quit(self, name=sessions.default):
         """Save open pages and quit.
 

qutebrowser/browser/commands.py

@@ -42,7 +42,7 @@ from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils,
                                objreg, utils, typing, debug)
 from qutebrowser.utils.usertypes import KeyMode
 from qutebrowser.misc import editor, guiprocess
-from qutebrowser.completion.models import instances, sortfilter
+from qutebrowser.completion.models import urlmodel, miscmodels
 
 
 class CommandDispatcher:
@@ -272,7 +272,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', name='open',
                        maxsplit=0, scope='window')
-    @cmdutils.argument('url', completion=usertypes.Completion.url)
+    @cmdutils.argument('url', completion=urlmodel.url)
     @cmdutils.argument('count', count=True)
     def openurl(self, url=None, implicit=False,
                 bg=False, tab=False, window=False, count=None, secure=False,
@@ -1010,7 +1010,7 @@ class CommandDispatcher:
             self._open(url, tab, bg, window)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    @cmdutils.argument('index', completion=usertypes.Completion.tab)
+    @cmdutils.argument('index', completion=miscmodels.buffer)
     def buffer(self, index):
         """Select tab by index or url/title best match.
 
@@ -1026,11 +1026,10 @@ class CommandDispatcher:
             for part in index_parts:
                 int(part)
         except ValueError:
-            model = instances.get(usertypes.Completion.tab)
+            model = miscmodels.buffer()
-            sf = sortfilter.CompletionFilterModel(source=model)
+            model.set_pattern(index)
-            sf.set_pattern(index)
+            if model.count() > 0:
-            if sf.count() > 0:
+                index = model.data(model.first_item())
-                index = sf.data(sf.first_item())
                 index_parts = index.split('/', 1)
             else:
                 raise cmdexc.CommandError(
@@ -1235,8 +1234,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0)
-    @cmdutils.argument('name',
+    @cmdutils.argument('name', completion=miscmodels.quickmark)
-                       completion=usertypes.Completion.quickmark_by_name)
     def quickmark_load(self, name, tab=False, bg=False, window=False):
         """Load a quickmark.
 
@@ -1254,8 +1252,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0)
-    @cmdutils.argument('name',
+    @cmdutils.argument('name', completion=miscmodels.quickmark)
-                       completion=usertypes.Completion.quickmark_by_name)
     def quickmark_del(self, name=None):
         """Delete a quickmark.
 
@@ -1317,7 +1314,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0)
-    @cmdutils.argument('url', completion=usertypes.Completion.bookmark_by_url)
+    @cmdutils.argument('url', completion=miscmodels.bookmark)
     def bookmark_load(self, url, tab=False, bg=False, window=False,
                       delete=False):
         """Load a bookmark.
@@ -1339,7 +1336,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0)
-    @cmdutils.argument('url', completion=usertypes.Completion.bookmark_by_url)
+    @cmdutils.argument('url', completion=miscmodels.bookmark)
     def bookmark_del(self, url=None):
         """Delete a bookmark.
 
@@ -1523,7 +1520,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', name='help',
                        scope='window')
-    @cmdutils.argument('topic', completion=usertypes.Completion.helptopic)
+    @cmdutils.argument('topic', completion=miscmodels.helptopic)
     def show_help(self, tab=False, bg=False, window=False, topic=None):
         r"""Show help about a command or setting.
 

qutebrowser/browser/history.py

@@ -19,214 +19,82 @@
 
 """Simple history which gets written to disk."""
 
+import os
 import time
-import collections
 
-from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject
+from PyQt5.QtCore import pyqtSlot, QUrl, QTimer
 
-from qutebrowser.commands import cmdutils
+from qutebrowser.commands import cmdutils, cmdexc
-from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils,
+from qutebrowser.utils import (utils, objreg, log, usertypes, message,
-                               usertypes, message)
+                               debug, standarddir)
-from qutebrowser.misc import lineparser, objects
+from qutebrowser.misc import objects, sql
 
 
-class Entry:
+class CompletionHistory(sql.SqlTable):
 
-    """A single entry in the web history.
+    """History which only has the newest entry for each URL."""
 
-    Attributes:
+    def __init__(self, parent=None):
-        atime: The time the page was accessed.
+        super().__init__("CompletionHistory", ['url', 'title', 'last_atime'],
-        url: The URL which was accessed as QUrl.
+                         constraints={'url': 'PRIMARY KEY'}, parent=parent)
-        redirect: If True, don't save this entry to disk
+        self.create_index('CompletionHistoryAtimeIndex', 'last_atime')
-    """
-
-    def __init__(self, atime, url, title, redirect=False):
-        self.atime = float(atime)
-        self.url = url
-        self.title = title
-        self.redirect = redirect
-        qtutils.ensure_valid(url)
-
-    def __repr__(self):
-        return utils.get_repr(self, constructor=True, atime=self.atime,
-                              url=self.url_str(), title=self.title,
-                              redirect=self.redirect)
-
-    def __str__(self):
-        atime = str(int(self.atime))
-        if self.redirect:
-            atime += '-r'  # redirect flag
-        elems = [atime, self.url_str()]
-        if self.title:
-            elems.append(self.title)
-        return ' '.join(elems)
-
-    def __eq__(self, other):
-        return (self.atime == other.atime and
-                self.title == other.title and
-                self.url == other.url and
-                self.redirect == other.redirect)
-
-    def url_str(self):
-        """Get the URL as a lossless string."""
-        return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)
-
-    @classmethod
-    def from_str(cls, line):
-        """Parse a history line like '12345 http://example.com title'."""
-        data = line.split(maxsplit=2)
-        if len(data) == 2:
-            atime, url = data
-            title = ""
-        elif len(data) == 3:
-            atime, url, title = data
-        else:
-            raise ValueError("2 or 3 fields expected")
-
-        url = QUrl(url)
-        if not url.isValid():
-            raise ValueError("Invalid URL: {}".format(url.errorString()))
-
-        # https://github.com/qutebrowser/qutebrowser/issues/670
-        atime = atime.lstrip('\0')
-
-        if '-' in atime:
-            atime, flags = atime.split('-')
-        else:
-            flags = ''
-
-        if not set(flags).issubset('r'):
-            raise ValueError("Invalid flags {!r}".format(flags))
-
-        redirect = 'r' in flags
-
-        return cls(atime, url, title, redirect=redirect)
 
 
-class WebHistory(QObject):
+class WebHistory(sql.SqlTable):
 
-    """The global history of visited pages.
+    """The global history of visited pages."""
 
-    This is a little more complex as you'd expect so the history can be read
+    def __init__(self, parent=None):
-    from disk async while new history is already arriving.
+        super().__init__("History", ['url', 'title', 'atime', 'redirect'],
+                         parent=parent)
+        self.completion = CompletionHistory(parent=self)
+        self.create_index('HistoryIndex', 'url')
+        self.create_index('HistoryAtimeIndex', 'atime')
+        self._contains_query = self.contains_query('url')
+        self._between_query = sql.Query('SELECT * FROM History '
+                                        'where not redirect '
+                                        'and not url like "qute://%" '
+                                        'and atime > :earliest '
+                                        'and atime <= :latest '
+                                        'ORDER BY atime desc')
 
-    self.history_dict is the main place where the history is stored, in an
+        self._before_query = sql.Query('SELECT * FROM History '
-    OrderedDict (sorted by time) of URL strings mapped to Entry objects.
+                                       'where not redirect '
-
+                                       'and not url like "qute://%" '
-    While reading from disk is still ongoing, the history is saved in
+                                       'and atime <= :latest '
-    self._temp_history instead, and then appended to self.history_dict once
+                                       'ORDER BY atime desc '
-    that's fully populated.
+                                       'limit :limit offset :offset')
-
-    All history which is new in this session (rather than read from disk from a
-    previous browsing session) is also stored in self._new_history.
-    self._saved_count tracks how many of those entries were already written to
-    disk, so we can always append to the existing data.
-
-    Attributes:
-        history_dict: An OrderedDict of URLs read from the on-disk history.
-        _lineparser: The AppendLineParser used to save the history.
-        _new_history: A list of Entry items of the current session.
-        _saved_count: How many HistoryEntries have been written to disk.
-        _initial_read_started: Whether async_read was called.
-        _initial_read_done: Whether async_read has completed.
-        _temp_history: OrderedDict of temporary history entries before
-                       async_read was called.
-
-    Signals:
-        add_completion_item: Emitted before a new Entry is added.
-                             Used to sync with the completion.
-                             arg: The new Entry.
-        item_added: Emitted after a new Entry is added.
-                    Used to tell the savemanager that the history is dirty.
-                    arg: The new Entry.
-        cleared: Emitted after the history is cleared.
-    """
-
-    add_completion_item = pyqtSignal(Entry)
-    item_added = pyqtSignal(Entry)
-    cleared = pyqtSignal()
-    async_read_done = pyqtSignal()
-
-    def __init__(self, hist_dir, hist_name, parent=None):
-        super().__init__(parent)
-        self._initial_read_started = False
-        self._initial_read_done = False
-        self._lineparser = lineparser.AppendLineParser(hist_dir, hist_name,
-                                                       parent=self)
-        self.history_dict = collections.OrderedDict()
-        self._temp_history = collections.OrderedDict()
-        self._new_history = []
-        self._saved_count = 0
-        objreg.get('save-manager').add_saveable(
-            'history', self.save, self.item_added)
 
     def __repr__(self):
         return utils.get_repr(self, length=len(self))
 
-    def __iter__(self):
+    def __contains__(self, url):
-        return iter(self.history_dict.values())
+        return self._contains_query.run(val=url).value()
-
-    def __len__(self):
-        return len(self.history_dict)
-
-    def async_read(self):
-        """Read the initial history."""
-        if self._initial_read_started:
-            log.init.debug("Ignoring async_read() because reading is started.")
-            return
-        self._initial_read_started = True
-
-        with self._lineparser.open():
-            for line in self._lineparser:
-                yield
-
-                line = line.rstrip()
-                if not line:
-                    continue
-
-                try:
-                    entry = Entry.from_str(line)
-                except ValueError as e:
-                    log.init.warning("Invalid history entry {!r}: {}!".format(
-                        line, e))
-                    continue
-
-                # This de-duplicates history entries; only the latest
-                # entry for each URL is kept. If you want to keep
-                # information about previous hits change the items in
-                # old_urls to be lists or change Entry to have a
-                # list of atimes.
-                self._add_entry(entry)
-
-        self._initial_read_done = True
-        self.async_read_done.emit()
-
-        for entry in self._temp_history.values():
-            self._add_entry(entry)
-            self._new_history.append(entry)
-            if not entry.redirect:
-                self.add_completion_item.emit(entry)
-        self._temp_history.clear()
-
-    def _add_entry(self, entry, target=None):
-        """Add an entry to self.history_dict or another given OrderedDict."""
-        if target is None:
-            target = self.history_dict
-        url_str = entry.url_str()
-        target[url_str] = entry
-        target.move_to_end(url_str)
 
     def get_recent(self):
         """Get the most recent history entries."""
-        old = self._lineparser.get_recent()
+        return self.select(sort_by='atime', sort_order='desc', limit=100)
-        return old + [str(e) for e in self._new_history]
 
-    def save(self):
+    def entries_between(self, earliest, latest):
-        """Save the history to disk."""
+        """Iterate non-redirect, non-qute entries between two timestamps.
-        new = (str(e) for e in self._new_history[self._saved_count:])
+
-        self._lineparser.new_data = new
+        Args:
-        self._lineparser.save()
+            earliest: Omit timestamps earlier than this.
-        self._saved_count = len(self._new_history)
+            latest: Omit timestamps later than this.
+        """
+        self._between_query.run(earliest=earliest, latest=latest)
+        return iter(self._between_query)
+
+    def entries_before(self, latest, limit, offset):
+        """Iterate non-redirect, non-qute entries occurring before a timestamp.
+
+        Args:
+            latest: Omit timestamps more recent than this.
+            limit: Max number of entries to include.
+            offset: Number of entries to skip.
+        """
+        self._before_query.run(latest=latest, limit=limit, offset=offset)
+        return iter(self._before_query)
 
     @cmdutils.register(name='history-clear', instance='web-history')
     def clear(self, force=False):
@@ -246,12 +114,17 @@ class WebHistory(QObject):
                                 "history?")
 
     def _do_clear(self):
-        self._lineparser.clear()
+        self.delete_all()
-        self.history_dict.clear()
+        self.completion.delete_all()
-        self._temp_history.clear()
+
-        self._new_history.clear()
+    def delete_url(self, url):
-        self._saved_count = 0
+        """Remove all history entries with the given url.
-        self.cleared.emit()
+
+        Args:
+            url: URL string to delete.
+        """
+        self.delete('url', url)
+        self.completion.delete('url', url)
 
     @pyqtSlot(QUrl, QUrl, str)
     def add_from_tab(self, url, requested_url, title):
@@ -285,17 +158,129 @@ class WebHistory(QObject):
             log.misc.warning("Ignoring invalid URL being added to history")
             return
 
-        if atime is None:
+        atime = int(atime) if (atime is not None) else int(time.time())
-            atime = time.time()
+        url_str = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)
-        entry = Entry(atime, url, title, redirect=redirect)
+        self.insert({'url': url_str,
-        if self._initial_read_done:
+                     'title': title,
-            self._add_entry(entry)
+                     'atime': atime,
-            self._new_history.append(entry)
+                     'redirect': redirect})
-            self.item_added.emit(entry)
+        if not redirect:
-            if not entry.redirect:
+            self.completion.insert({'url': url_str,
-                self.add_completion_item.emit(entry)
+                                    'title': title,
+                                    'last_atime': atime},
+                                   replace=True)
+
+    def _parse_entry(self, line):
+        """Parse a history line like '12345 http://example.com title'."""
+        if not line or line.startswith('#'):
+            return None
+        data = line.split(maxsplit=2)
+        if len(data) == 2:
+            atime, url = data
+            title = ""
+        elif len(data) == 3:
+            atime, url, title = data
         else:
-            self._add_entry(entry, target=self._temp_history)
+            raise ValueError("2 or 3 fields expected")
+
+        # http://xn--pple-43d.com/ with
+        # https://bugreports.qt.io/browse/QTBUG-60364
+        if url in ['http://.com/', 'https://www..com/']:
+            return None
+
+        url = QUrl(url)
+        if not url.isValid():
+            raise ValueError("Invalid URL: {}".format(url.errorString()))
+
+        # https://github.com/qutebrowser/qutebrowser/issues/2646
+        if url.scheme() == 'data':
+            return None
+
+        # https://github.com/qutebrowser/qutebrowser/issues/670
+        atime = atime.lstrip('\0')
+
+        if '-' in atime:
+            atime, flags = atime.split('-')
+        else:
+            flags = ''
+
+        if not set(flags).issubset('r'):
+            raise ValueError("Invalid flags {!r}".format(flags))
+
+        redirect = 'r' in flags
+        return (url, title, int(atime), redirect)
+
+    def import_txt(self):
+        """Import a history text file into sqlite if it exists.
+
+        In older versions of qutebrowser, history was stored in a text format.
+        This converts that file into the new sqlite format and moves it to a
+        backup location.
+        """
+        path = os.path.join(standarddir.data(), 'history')
+        if not os.path.isfile(path):
+            return
+
+        def action():
+            with debug.log_time(log.init, 'Import old history file to sqlite'):
+                try:
+                    self._read(path)
+                except ValueError as ex:
+                    message.error('Failed to import history: {}'.format(ex))
+                else:
+                    bakpath = path + '.bak'
+                    message.info('History import complete. Moving {} to {}'
+                                 .format(path, bakpath))
+                    os.rename(path, bakpath)
+
+        # delay to give message time to appear before locking down for import
+        message.info('Converting {} to sqlite...'.format(path))
+        QTimer.singleShot(100, action)
+
+    def _read(self, path):
+        """Import a text file into the sql database."""
+        with open(path, 'r', encoding='utf-8') as f:
+            data = {'url': [], 'title': [], 'atime': [], 'redirect': []}
+            completion_data = {'url': [], 'title': [], 'last_atime': []}
+            for (i, line) in enumerate(f):
+                try:
+                    parsed = self._parse_entry(line.strip())
+                    if parsed is None:
+                        continue
+                    url, title, atime, redirect = parsed
+                    data['url'].append(url)
+                    data['title'].append(title)
+                    data['atime'].append(atime)
+                    data['redirect'].append(redirect)
+                    if not redirect:
+                        completion_data['url'].append(url)
+                        completion_data['title'].append(title)
+                        completion_data['last_atime'].append(atime)
+                except ValueError as ex:
+                    raise ValueError('Failed to parse line #{} of {}: "{}"'
+                                     .format(i, path, ex))
+        self.insert_batch(data)
+        self.completion.insert_batch(completion_data, replace=True)
+
+    @cmdutils.register(instance='web-history', debug=True)
+    def debug_dump_history(self, dest):
+        """Dump the history to a file in the old pre-SQL format.
+
+        Args:
+            dest: Where to write the file to.
+        """
+        dest = os.path.expanduser(dest)
+
+        lines = ('{}{} {} {}'
+                 .format(int(x.atime), '-r' * x.redirect, x.url, x.title)
+                 for x in self.select(sort_by='atime', sort_order='asc'))
+
+        try:
+            with open(dest, 'w', encoding='utf-8') as f:
+                f.write('\n'.join(lines))
+            message.info("Dumped history to {}".format(dest))
+        except OSError as e:
+            raise cmdexc.CommandError('Could not write history: {}', e)
 
 
 def init(parent=None):
@@ -304,8 +289,7 @@ def init(parent=None):
     Args:
         parent: The parent to use for WebHistory.
     """
-    history = WebHistory(hist_dir=standarddir.data(), hist_name='history',
+    history = WebHistory(parent=parent)
-                         parent=parent)
     objreg.register('web-history', history)
 
     if objects.backend == usertypes.Backend.QtWebKit:

qutebrowser/browser/qutescheme.py

@@ -26,7 +26,6 @@ Module attributes:
 
 import json
 import os
-import sys
 import time
 import urllib.parse
 import datetime
@@ -186,88 +185,36 @@ def qute_bookmarks(_url):
     return 'text/html', html
 
 
-def history_data(start_time):  # noqa
+def history_data(start_time, offset=None):
-    """Return history data
+    """Return history data.
 
     Arguments:
-        start_time -- select history starting from this timestamp.
+        start_time: select history starting from this timestamp.
+        offset: number of items to skip
     """
-    def history_iter(start_time, reverse=False):
+    # history atimes are stored as ints, ensure start_time is not a float
-        """Iterate through the history and get items we're interested.
+    start_time = int(start_time)
-
+    hist = objreg.get('web-history')
-        Arguments:
+    if offset is not None:
-            reverse -- whether to reverse the history_dict before iterating.
+        entries = hist.entries_before(start_time, limit=1000, offset=offset)
-        """
+    else:
-        history = objreg.get('web-history').history_dict.values()
-        if reverse:
-            history = reversed(history)
-
-        # when history_dict is not reversed, we need to keep track of last item
-        # so that we can yield its atime
-        last_item = None
-
         # end is 24hrs earlier than start
         end_time = start_time - 24*60*60
+        entries = hist.entries_between(end_time, start_time)
 
-        for item in history:
+    return [{"url": e.url, "title": e.title or e.url, "time": e.atime}
-            # Skip redirects
+            for e in entries]
-            # Skip qute:// links
-            if item.redirect or item.url.scheme() == 'qute':
-                continue
-
-            # Skip items out of time window
-            item_newer = item.atime > start_time
-            item_older = item.atime <= end_time
-            if reverse:
-                # history_dict is reversed, we are going back in history.
-                # so:
-                #     abort if item is older than start_time+24hr
-                #     skip if item is newer than start
-                if item_older:
-                    yield {"next": int(item.atime)}
-                    return
-                if item_newer:
-                    continue
-            else:
-                # history_dict isn't reversed, we are going forward in history.
-                # so:
-                #     abort if item is newer than start_time
-                #     skip if item is older than start_time+24hrs
-                if item_older:
-                    last_item = item
-                    continue
-                if item_newer:
-                    yield {"next": int(last_item.atime if last_item else -1)}
-                    return
-
-            # Use item's url as title if there's no title.
-            item_url = item.url.toDisplayString()
-            item_title = item.title if item.title else item_url
-            item_time = int(item.atime * 1000)
-
-            yield {"url": item_url, "title": item_title, "time": item_time}
-
-        # if we reached here, we had reached the end of history
-        yield {"next": int(last_item.atime if last_item else -1)}
-
-    if sys.hexversion >= 0x03050000:
-        # On Python >= 3.5 we can reverse the ordereddict in-place and thus
-        # apply an additional performance improvement in history_iter.
-        # On my machine, this gets us down from 550ms to 72us with 500k old
-        # items.
-        history = history_iter(start_time, reverse=True)
-    else:
-        # On Python 3.4, we can't do that, so we'd need to copy the entire
-        # history to a list. There, filter first and then reverse it here.
-        history = reversed(list(history_iter(start_time, reverse=False)))
-
-    return list(history)
 
 
 @add_handler('history')
 def qute_history(url):
     """Handler for qute://history. Display and serve history."""
     if url.path() == '/data':
+        try:
+            offset = QUrlQuery(url).queryItemValue("offset")
+            offset = int(offset) if offset else None
+        except ValueError as e:
+            raise QuteSchemeError("Query parameter offset is invalid", e)
         # Use start_time in query or current time.
         try:
             start_time = QUrlQuery(url).queryItemValue("start_time")
@@ -275,7 +222,7 @@ def qute_history(url):
         except ValueError as e:
             raise QuteSchemeError("Query parameter start_time is invalid", e)
 
-        return 'text/html', json.dumps(history_data(start_time))
+        return 'text/html', json.dumps(history_data(start_time, offset))
     else:
         if (
             config.get('content', 'allow-javascript') and
@@ -307,9 +254,9 @@ def qute_history(url):
             start_time = time.mktime(next_date.timetuple()) - 1
             history = [
                 (i["url"], i["title"],
-                 datetime.datetime.fromtimestamp(i["time"]/1000),
+                 datetime.datetime.fromtimestamp(i["time"]),
                  QUrl(i["url"]).host())
-                for i in history_data(start_time) if "next" not in i
+                for i in history_data(start_time)
             ]
 
             return 'text/html', jinja.render(

qutebrowser/browser/webkit/webkithistory.py

@@ -19,9 +19,12 @@
 
 """QtWebKit specific part of history."""
 
+import functools
 
 from PyQt5.QtWebKit import QWebHistoryInterface
 
+from qutebrowser.utils import debug
+
 
 class WebHistoryInterface(QWebHistoryInterface):
 
@@ -34,11 +37,13 @@ class WebHistoryInterface(QWebHistoryInterface):
     def __init__(self, webhistory, parent=None):
         super().__init__(parent)
         self._history = webhistory
+        self._history.changed.connect(self.historyContains.cache_clear)
 
     def addHistoryEntry(self, url_string):
         """Required for a QWebHistoryInterface impl, obsoleted by add_url."""
         pass
 
+    @functools.lru_cache(maxsize=32768)
     def historyContains(self, url_string):
         """Called by WebKit to determine if a URL is contained in the history.
 
@@ -48,7 +53,8 @@ class WebHistoryInterface(QWebHistoryInterface):
         Return:
             True if the url is in the history, False otherwise.
         """
-        return url_string in self._history.history_dict
+        with debug.log_time('sql', 'historyContains'):
+            return url_string in self._history
 
 
 def init(history):

qutebrowser/completion/completer.py

@@ -23,8 +23,8 @@ from PyQt5.QtCore import pyqtSlot, QObject, QTimer
 
 from qutebrowser.config import config
 from qutebrowser.commands import cmdutils, runners
-from qutebrowser.utils import usertypes, log, utils
+from qutebrowser.utils import log, utils, debug
-from qutebrowser.completion.models import instances, sortfilter
+from qutebrowser.completion.models import miscmodels
 
 
 class Completer(QObject):
@@ -39,6 +39,7 @@ class Completer(QObject):
         _last_cursor_pos: The old cursor position so we avoid double completion
                           updates.
         _last_text: The old command text so we avoid double completion updates.
+        _last_completion_func: The completion function used for the last text.
     """
 
     def __init__(self, cmd, win_id, parent=None):
@@ -52,6 +53,7 @@ class Completer(QObject):
         self._timer.timeout.connect(self._update_completion)
         self._last_cursor_pos = None
         self._last_text = None
+        self._last_completion_func = None
         self._cmd.update_completion.connect(self.schedule_completion_update)
 
     def __repr__(self):
@@ -62,37 +64,8 @@ class Completer(QObject):
         completion = self.parent()
         return completion.model()
 
-    def _get_completion_model(self, completion, pos_args):
-        """Get a completion model based on an enum member.
-
-        Args:
-            completion: A usertypes.Completion member.
-            pos_args: The positional args entered before the cursor.
-
-        Return:
-            A completion model or None.
-        """
-        if completion == usertypes.Completion.option:
-            section = pos_args[0]
-            model = instances.get(completion).get(section)
-        elif completion == usertypes.Completion.value:
-            section = pos_args[0]
-            option = pos_args[1]
-            try:
-                model = instances.get(completion)[section][option]
-            except KeyError:
-                # No completion model for this section/option.
-                model = None
-        else:
-            model = instances.get(completion)
-
-        if model is None:
-            return None
-        else:
-            return sortfilter.CompletionFilterModel(source=model, parent=self)
-
     def _get_new_completion(self, before_cursor, under_cursor):
-        """Get a new completion.
+        """Get the completion function based on the current command text.
 
         Args:
             before_cursor: The command chunks before the cursor.
@@ -109,8 +82,8 @@ class Completer(QObject):
         log.completion.debug("After removing flags: {}".format(before_cursor))
         if not before_cursor:
             # '|' or 'set|'
-            model = instances.get(usertypes.Completion.command)
+            log.completion.debug('Starting command completion')
-            return sortfilter.CompletionFilterModel(source=model, parent=self)
+            return miscmodels.command
         try:
             cmd = cmdutils.cmd_dict[before_cursor[0]]
         except KeyError:
@@ -119,14 +92,11 @@ class Completer(QObject):
             return None
         argpos = len(before_cursor) - 1
         try:
-            completion = cmd.get_pos_arg_info(argpos).completion
+            func = cmd.get_pos_arg_info(argpos).completion
         except IndexError:
             log.completion.debug("No completion in position {}".format(argpos))
             return None
-        if completion is None:
+        return func
-            return None
-        model = self._get_completion_model(completion, before_cursor[1:])
-        return model
 
     def _quote(self, s):
         """Quote s if it needs quoting for the commandline.
@@ -241,6 +211,7 @@ class Completer(QObject):
             # FIXME complete searches
             # https://github.com/qutebrowser/qutebrowser/issues/32
             completion.set_model(None)
+            self._last_completion_func = None
             return
 
         before_cursor, pattern, after_cursor = self._partition()
@@ -249,13 +220,24 @@ class Completer(QObject):
             before_cursor, pattern, after_cursor))
 
         pattern = pattern.strip("'\"")
-        model = self._get_new_completion(before_cursor, pattern)
+        func = self._get_new_completion(before_cursor, pattern)
 
-        log.completion.debug("Setting completion model to {} with pattern '{}'"
+        if func is None:
-            .format(model.srcmodel.__class__.__name__ if model else 'None',
+            log.completion.debug('Clearing completion')
-                    pattern))
+            completion.set_model(None)
+            self._last_completion_func = None
+            return
 
-        completion.set_model(model, pattern)
+        if func != self._last_completion_func:
+            self._last_completion_func = func
+            args = (x for x in before_cursor[1:] if not x.startswith('-'))
+            with debug.log_time(log.completion,
+                    'Starting {} completion'.format(func.__name__)):
+                model = func(*args)
+            with debug.log_time(log.completion, 'Set completion model'):
+                completion.set_model(model)
+
+        completion.set_pattern(pattern)
 
     def _change_completed_part(self, newtext, before, after, immediate=False):
         """Change the part we're currently completing in the commandline.

qutebrowser/completion/completiondelegate.py

@@ -196,8 +196,9 @@ class CompletionItemDelegate(QStyledItemDelegate):
         self._doc.setDocumentMargin(2)
 
         if index.parent().isValid():
-            pattern = index.model().pattern
+            view = self.parent()
-            columns_to_filter = index.model().srcmodel.columns_to_filter
+            pattern = view.pattern
+            columns_to_filter = index.model().columns_to_filter(index)
             if index.column() in columns_to_filter and pattern:
                 repl = r'<span class="highlight">\g<0></span>'
                 text = re.sub(re.escape(pattern).replace(r'\ ', r'|'),

qutebrowser/completion/completionwidget.py

@@ -28,8 +28,7 @@ from PyQt5.QtCore import pyqtSlot, pyqtSignal, Qt, QItemSelectionModel, QSize
 
 from qutebrowser.config import config, style
 from qutebrowser.completion import completiondelegate
-from qutebrowser.completion.models import base
+from qutebrowser.utils import utils, usertypes, objreg, debug, log
-from qutebrowser.utils import utils, usertypes, objreg
 from qutebrowser.commands import cmdexc, cmdutils
 
 
@@ -41,6 +40,7 @@ class CompletionView(QTreeView):
     headers, and children show as flat list.
 
     Attributes:
+        pattern: Current filter pattern, used for highlighting.
         _win_id: The ID of the window this CompletionView is associated with.
         _height: The height to use for the CompletionView.
         _height_perc: Either None or a percentage if height should be relative.
@@ -107,12 +107,12 @@ class CompletionView(QTreeView):
 
     def __init__(self, win_id, parent=None):
         super().__init__(parent)
+        self.pattern = ''
         self._win_id = win_id
         # FIXME handle new aliases.
         # objreg.get('config').changed.connect(self.init_command_completion)
         objreg.get('config').changed.connect(self._on_config_changed)
 
-        self._column_widths = base.BaseCompletionModel.COLUMN_WIDTHS
         self._active = False
 
         self._delegate = completiondelegate.CompletionItemDelegate(self)
@@ -151,7 +151,8 @@ class CompletionView(QTreeView):
     def _resize_columns(self):
         """Resize the completion columns based on column_widths."""
         width = self.size().width()
-        pixel_widths = [(width * perc // 100) for perc in self._column_widths]
+        column_widths = self.model().column_widths
+        pixel_widths = [(width * perc // 100) for perc in column_widths]
 
         if self.verticalScrollBar().isVisible():
             delta = self.style().pixelMetric(QStyle.PM_ScrollBarExtent) + 5
@@ -262,47 +263,49 @@ class CompletionView(QTreeView):
         elif config.get('completion', 'show') == 'auto':
             self.show()
 
-    def set_model(self, model, pattern=None):
+    def set_model(self, model):
         """Switch completion to a new model.
 
         Called from on_update_completion().
 
         Args:
             model: The model to use.
-            pattern: The filter pattern to set (what the user entered).
         """
+        if self.model() is not None and model is not self.model():
+            self.model().deleteLater()
+            self.selectionModel().deleteLater()
+
+        self.setModel(model)
+
         if model is None:
             self._active = False
             self.hide()
             return
 
-        old_model = self.model()
+        model.setParent(self)
-        if model is not old_model:
-            sel_model = self.selectionModel()
-
-            self.setModel(model)
         self._active = True
+        self._maybe_show()
 
-            if sel_model is not None:
+        self._resize_columns()
-                sel_model.deleteLater()
-            if old_model is not None:
-                old_model.deleteLater()
-
-        if (config.get('completion', 'show') == 'always' and
-                model.count() > 0):
-            self.show()
-        else:
-            self.hide()
-
         for i in range(model.rowCount()):
             self.expand(model.index(i, 0))
 
-        if pattern is not None:
+    def set_pattern(self, pattern):
-            model.set_pattern(pattern)
+        """Set the pattern on the underlying model."""
-
+        if not self.model():
-        self._column_widths = model.srcmodel.COLUMN_WIDTHS
+            return
-        self._resize_columns()
+        self.pattern = pattern
+        with debug.log_time(log.completion, 'Set pattern {}'.format(pattern)):
+            self.model().set_pattern(pattern)
             self._maybe_update_geometry()
+            self._maybe_show()
+
+    def _maybe_show(self):
+        if (config.get('completion', 'show') == 'always' and
+                self.model().count() > 0):
+            self.show()
+        else:
+            self.hide()
 
     def _maybe_update_geometry(self):
         """Emit the update_geometry signal if the config says so."""
@@ -347,7 +350,7 @@ class CompletionView(QTreeView):
         indexes = selected.indexes()
         if not indexes:
             return
-        data = self.model().data(indexes[0])
+        data = str(self.model().data(indexes[0]))
         self.selection_changed.emit(data)
 
     def resizeEvent(self, e):
@@ -367,9 +370,7 @@ class CompletionView(QTreeView):
                        modes=[usertypes.KeyMode.command], scope='window')
     def completion_item_del(self):
         """Delete the current completion item."""
-        if not self.currentIndex().isValid():
+        index = self.currentIndex()
+        if not index.isValid():
             raise cmdexc.CommandError("No item selected!")
-        try:
+        self.model().delete_cur_item(index)
-            self.model().srcmodel.delete_cur_item(self)
-        except NotImplementedError:
-            raise cmdexc.CommandError("Cannot delete this item.")

qutebrowser/completion/models/base.py

@@ -1,130 +0,0 @@
-# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
-
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-"""The base completion model for completion in the command line.
-
-Module attributes:
-    Role: An enum of user defined model roles.
-"""
-
-from PyQt5.QtCore import Qt
-from PyQt5.QtGui import QStandardItemModel, QStandardItem
-
-from qutebrowser.utils import usertypes
-
-
-Role = usertypes.enum('Role', ['sort', 'userdata'], start=Qt.UserRole,
-                      is_int=True)
-
-
-class BaseCompletionModel(QStandardItemModel):
-
-    """A simple QStandardItemModel adopted for completions.
-
-    Used for showing completions later in the CompletionView. Supports setting
-    marks and adding new categories/items easily.
-
-    Class Attributes:
-        COLUMN_WIDTHS: The width percentages of the columns used in the
-                        completion view.
-        DUMB_SORT: the dumb sorting used by the model
-    """
-
-    COLUMN_WIDTHS = (30, 70, 0)
-    DUMB_SORT = None
-
-    def __init__(self, parent=None):
-        super().__init__(parent)
-        self.setColumnCount(3)
-        self.columns_to_filter = [0]
-
-    def new_category(self, name, sort=None):
-        """Add a new category to the model.
-
-        Args:
-            name: The name of the category to add.
-            sort: The value to use for the sort role.
-
-        Return:
-            The created QStandardItem.
-        """
-        cat = QStandardItem(name)
-        if sort is not None:
-            cat.setData(sort, Role.sort)
-        self.appendRow(cat)
-        return cat
-
-    def new_item(self, cat, name, desc='', misc=None, sort=None,
-                 userdata=None):
-        """Add a new item to a category.
-
-        Args:
-            cat: The parent category.
-            name: The name of the item.
-            desc: The description of the item.
-            misc: Misc text to display.
-            sort: Data for the sort role (int).
-            userdata: User data to be added for the first column.
-
-        Return:
-            A (nameitem, descitem, miscitem) tuple.
-        """
-        assert not isinstance(name, int)
-        assert not isinstance(desc, int)
-        assert not isinstance(misc, int)
-
-        nameitem = QStandardItem(name)
-        descitem = QStandardItem(desc)
-        if misc is None:
-            miscitem = QStandardItem()
-        else:
-            miscitem = QStandardItem(misc)
-
-        cat.appendRow([nameitem, descitem, miscitem])
-        if sort is not None:
-            nameitem.setData(sort, Role.sort)
-        if userdata is not None:
-            nameitem.setData(userdata, Role.userdata)
-        return nameitem, descitem, miscitem
-
-    def delete_cur_item(self, completion):
-        """Delete the selected item."""
-        raise NotImplementedError
-
-    def flags(self, index):
-        """Return the item flags for index.
-
-        Override QAbstractItemModel::flags.
-
-        Args:
-            index: The QModelIndex to get item flags for.
-
-        Return:
-            The item flags, or Qt.NoItemFlags on error.
-        """
-        if not index.isValid():
-            return
-
-        if index.parent().isValid():
-            # item
-            return (Qt.ItemIsEnabled | Qt.ItemIsSelectable |
-                    Qt.ItemNeverHasChildren)
-        else:
-            # category
-            return Qt.NoItemFlags

qutebrowser/completion/models/completionmodel.py

@@ -0,0 +1,224 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Ryan Roden-Corrent (rcorre) <ryan@rcorre.net>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""A model that proxies access to one or more completion categories."""
+
+from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel
+
+from qutebrowser.utils import log, qtutils
+
+
+class CompletionModel(QAbstractItemModel):
+
+    """A model that proxies access to one or more completion categories.
+
+    Top level indices represent categories.
+    Child indices represent rows of those tables.
+
+    Attributes:
+        column_widths: The width percentages of the columns used in the
+                       completion view.
+        _categories: The sub-categories.
+    """
+
+    def __init__(self, *, column_widths=(30, 70, 0), parent=None):
+        super().__init__(parent)
+        self.column_widths = column_widths
+        self._categories = []
+
+    def _cat_from_idx(self, index):
+        """Return the category pointed to by the given index.
+
+        Args:
+            idx: A QModelIndex
+        Returns:
+            A category if the index points at one, else None
+        """
+        # items hold an index to the parent category in their internalPointer
+        # categories have an empty internalPointer
+        if index.isValid() and not index.internalPointer():
+            return self._categories[index.row()]
+        return None
+
+    def add_category(self, cat):
+        """Add a completion category to the model."""
+        self._categories.append(cat)
+        cat.layoutAboutToBeChanged.connect(self.layoutAboutToBeChanged)
+        cat.layoutChanged.connect(self.layoutChanged)
+
+    def data(self, index, role=Qt.DisplayRole):
+        """Return the item data for index.
+
+        Override QAbstractItemModel::data.
+
+        Args:
+            index: The QModelIndex to get item flags for.
+
+        Return: The item data, or None on an invalid index.
+        """
+        if role != Qt.DisplayRole:
+            return None
+        cat = self._cat_from_idx(index)
+        if cat:
+            # category header
+            if index.column() == 0:
+                return self._categories[index.row()].name
+            return None
+        # item
+        cat = self._cat_from_idx(index.parent())
+        if not cat:
+            return None
+        idx = cat.index(index.row(), index.column())
+        return cat.data(idx)
+
+    def flags(self, index):
+        """Return the item flags for index.
+
+        Override QAbstractItemModel::flags.
+
+        Return: The item flags, or Qt.NoItemFlags on error.
+        """
+        if not index.isValid():
+            return Qt.NoItemFlags
+        if index.parent().isValid():
+            # item
+            return (Qt.ItemIsEnabled | Qt.ItemIsSelectable |
+                    Qt.ItemNeverHasChildren)
+        else:
+            # category
+            return Qt.NoItemFlags
+
+    def index(self, row, col, parent=QModelIndex()):
+        """Get an index into the model.
+
+        Override QAbstractItemModel::index.
+
+        Return: A QModelIndex.
+        """
+        if (row < 0 or row >= self.rowCount(parent) or
+                col < 0 or col >= self.columnCount(parent)):
+            return QModelIndex()
+        if parent.isValid():
+            if parent.column() != 0:
+                return QModelIndex()
+            # store a pointer to the parent category in internalPointer
+            return self.createIndex(row, col, self._categories[parent.row()])
+        return self.createIndex(row, col, None)
+
+    def parent(self, index):
+        """Get an index to the parent of the given index.
+
+        Override QAbstractItemModel::parent.
+
+        Args:
+            index: The QModelIndex to get the parent index for.
+        """
+        parent_cat = index.internalPointer()
+        if not parent_cat:
+            # categories have no parent
+            return QModelIndex()
+        row = self._categories.index(parent_cat)
+        return self.createIndex(row, 0, None)
+
+    def rowCount(self, parent=QModelIndex()):
+        """Override QAbstractItemModel::rowCount."""
+        if not parent.isValid():
+            # top-level
+            return len(self._categories)
+        cat = self._cat_from_idx(parent)
+        if not cat or parent.column() != 0:
+            # item or nonzero category column (only first col has children)
+            return 0
+        else:
+            # category
+            return cat.rowCount()
+
+    def columnCount(self, parent=QModelIndex()):
+        """Override QAbstractItemModel::columnCount."""
+        # pylint: disable=unused-argument
+        return 3
+
+    def canFetchMore(self, parent):
+        """Override to forward the call to the categories."""
+        cat = self._cat_from_idx(parent)
+        if cat:
+            return cat.canFetchMore(QModelIndex())
+        return False
+
+    def fetchMore(self, parent):
+        """Override to forward the call to the categories."""
+        cat = self._cat_from_idx(parent)
+        if cat:
+            cat.fetchMore(QModelIndex())
+
+    def count(self):
+        """Return the count of non-category items."""
+        return sum(t.rowCount() for t in self._categories)
+
+    def set_pattern(self, pattern):
+        """Set the filter pattern for all categories.
+
+        Args:
+            pattern: The filter pattern to set.
+        """
+        log.completion.debug("Setting completion pattern '{}'".format(pattern))
+        for cat in self._categories:
+            cat.set_pattern(pattern)
+
+    def first_item(self):
+        """Return the index of the first child (non-category) in the model."""
+        for row, cat in enumerate(self._categories):
+            if cat.rowCount() > 0:
+                parent = self.index(row, 0)
+                index = self.index(0, 0, parent)
+                qtutils.ensure_valid(index)
+                return index
+        return QModelIndex()
+
+    def last_item(self):
+        """Return the index of the last child (non-category) in the model."""
+        for row, cat in reversed(list(enumerate(self._categories))):
+            childcount = cat.rowCount()
+            if childcount > 0:
+                parent = self.index(row, 0)
+                index = self.index(childcount - 1, 0, parent)
+                qtutils.ensure_valid(index)
+                return index
+        return QModelIndex()
+
+    def columns_to_filter(self, index):
+        """Return the column indices the filter pattern applies to.
+
+        Args:
+            index: index of the item to check.
+
+        Return: A list of integers.
+        """
+        cat = self._cat_from_idx(index.parent())
+        return cat.columns_to_filter if cat else []
+
+    def delete_cur_item(self, index):
+        """Delete the row at the given index."""
+        qtutils.ensure_valid(index)
+        parent = index.parent()
+        cat = self._cat_from_idx(parent)
+        assert cat, "CompletionView sent invalid index for deletion"
+        self.beginRemoveRows(parent, index.row(), index.row())
+        cat.delete_cur_item(cat.index(index.row(), 0))
+        self.endRemoveRows()

qutebrowser/completion/models/configmodel.py

@@ -17,53 +17,34 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""CompletionModels for the config."""
+"""Functions that return config-related completion models."""
 
-from PyQt5.QtCore import pyqtSlot, Qt
+from qutebrowser.config import configdata, configexc
-
+from qutebrowser.completion.models import completionmodel, listcategory
-from qutebrowser.config import config, configdata
+from qutebrowser.utils import objreg
-from qutebrowser.utils import log, qtutils, objreg
-from qutebrowser.completion.models import base
 
 
-class SettingSectionCompletionModel(base.BaseCompletionModel):
+def section():
-
     """A CompletionModel filled with settings sections."""
-
+    model = completionmodel.CompletionModel(column_widths=(20, 70, 10))
-    # https://github.com/qutebrowser/qutebrowser/issues/545
+    sections = ((name, configdata.SECTION_DESC[name].splitlines()[0].strip())
-    # pylint: disable=abstract-method
+                for name in configdata.DATA)
-
+    model.add_category(listcategory.ListCategory("Sections", sections))
-    COLUMN_WIDTHS = (20, 70, 10)
+    return model
-
-    def __init__(self, parent=None):
-        super().__init__(parent)
-        cat = self.new_category("Sections")
-        for name in configdata.DATA:
-            desc = configdata.SECTION_DESC[name].splitlines()[0].strip()
-            self.new_item(cat, name, desc)
 
 
-class SettingOptionCompletionModel(base.BaseCompletionModel):
+def option(sectname):
-
     """A CompletionModel filled with settings and their descriptions.
 
-    Attributes:
+    Args:
-        _misc_items: A dict of the misc. column items which will be set later.
+        sectname: The name of the config section this model shows.
-        _section: The config section this model shows.
     """
-
+    model = completionmodel.CompletionModel(column_widths=(20, 70, 10))
-    # https://github.com/qutebrowser/qutebrowser/issues/545
+    try:
-    # pylint: disable=abstract-method
+        sectdata = configdata.DATA[sectname]
-
+    except KeyError:
-    COLUMN_WIDTHS = (20, 70, 10)
+        return None
-
+    options = []
-    def __init__(self, section, parent=None):
-        super().__init__(parent)
-        cat = self.new_category(section)
-        sectdata = configdata.DATA[section]
-        self._misc_items = {}
-        self._section = section
-        objreg.get('config').changed.connect(self.update_misc_column)
     for name in sectdata:
         try:
             desc = sectdata.descriptions[name]
@@ -73,86 +54,43 @@ class SettingOptionCompletionModel(base.BaseCompletionModel):
             desc = ""
         else:
             desc = desc.splitlines()[0]
-            value = config.get(section, name, raw=True)
+        config = objreg.get('config')
-            _valitem, _descitem, miscitem = self.new_item(cat, name, desc,
+        val = config.get(sectname, name, raw=True)
-                                                          value)
+        options.append((name, desc, val))
-            self._misc_items[name] = miscitem
+    model.add_category(listcategory.ListCategory(sectname, options))
-
+    return model
-    @pyqtSlot(str, str)
-    def update_misc_column(self, section, option):
-        """Update misc column when config changed."""
-        if section != self._section:
-            return
-        try:
-            item = self._misc_items[option]
-        except KeyError:
-            log.completion.debug("Couldn't get item {}.{} from model!".format(
-                section, option))
-            # changed before init
-            return
-        val = config.get(section, option, raw=True)
-        idx = item.index()
-        qtutils.ensure_valid(idx)
-        ok = self.setData(idx, val, Qt.DisplayRole)
-        if not ok:
-            raise ValueError("Setting data failed! (section: {}, option: {}, "
-                             "value: {})".format(section, option, val))
 
 
-class SettingValueCompletionModel(base.BaseCompletionModel):
+def value(sectname, optname):
-
     """A CompletionModel filled with setting values.
 
-    Attributes:
+    Args:
-        _section: The config section this model shows.
+        sectname: The name of the config section this model shows.
-        _option: The config option this model shows.
+        optname: The name of the config option this model shows.
     """
+    model = completionmodel.CompletionModel(column_widths=(20, 70, 10))
+    config = objreg.get('config')
 
-    # https://github.com/qutebrowser/qutebrowser/issues/545
+    try:
-    # pylint: disable=abstract-method
+        current = config.get(sectname, optname, raw=True) or '""'
+    except (configexc.NoSectionError, configexc.NoOptionError):
+        return None
 
-    COLUMN_WIDTHS = (20, 70, 10)
+    default = configdata.DATA[sectname][optname].default() or '""'
 
-    def __init__(self, section, option, parent=None):
+    if hasattr(configdata.DATA[sectname], 'valtype'):
-        super().__init__(parent)
-        self._section = section
-        self._option = option
-        objreg.get('config').changed.connect(self.update_current_value)
-        cur_cat = self.new_category("Current/Default", sort=0)
-        value = config.get(section, option, raw=True)
-        if not value:
-            value = '""'
-        self.cur_item, _descitem, _miscitem = self.new_item(cur_cat, value,
-                                                            "Current value")
-        default_value = configdata.DATA[section][option].default()
-        if not default_value:
-            default_value = '""'
-        self.new_item(cur_cat, default_value, "Default value")
-        if hasattr(configdata.DATA[section], 'valtype'):
         # Same type for all values (ValueList)
-            vals = configdata.DATA[section].valtype.complete()
+        vals = configdata.DATA[sectname].valtype.complete()
     else:
-            if option is None:
+        if optname is None:
-                raise ValueError("option may only be None for ValueList "
+            raise ValueError("optname may only be None for ValueList "
-                                 "sections, but {} is not!".format(section))
+                             "sections, but {} is not!".format(sectname))
         # Different type for each value (KeyValue)
-            vals = configdata.DATA[section][option].typ.complete()
+        vals = configdata.DATA[sectname][optname].typ.complete()
-        if vals is not None:
-            cat = self.new_category("Completions", sort=1)
-            for (val, desc) in vals:
-                self.new_item(cat, val, desc)
 
-    @pyqtSlot(str, str)
+    cur_cat = listcategory.ListCategory("Current/Default",
-    def update_current_value(self, section, option):
+        [(current, "Current value"), (default, "Default value")])
-        """Update current value when config changed."""
+    model.add_category(cur_cat)
-        if (section, option) != (self._section, self._option):
+    if vals is not None:
-            return
+        model.add_category(listcategory.ListCategory("Completions", vals))
-        value = config.get(section, option, raw=True)
+    return model
-        if not value:
-            value = '""'
-        idx = self.cur_item.index()
-        qtutils.ensure_valid(idx)
-        ok = self.setData(idx, value, Qt.DisplayRole)
-        if not ok:
-            raise ValueError("Setting data failed! (section: {}, option: {}, "
-                             "value: {})".format(section, option, value))

qutebrowser/completion/models/histcategory.py

@@ -0,0 +1,100 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Ryan Roden-Corrent (rcorre) <ryan@rcorre.net>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""A completion category that queries the SQL History store."""
+
+import re
+
+from PyQt5.QtSql import QSqlQueryModel
+
+from qutebrowser.misc import sql
+from qutebrowser.utils import debug
+from qutebrowser.commands import cmdexc
+from qutebrowser.config import config
+
+
+class HistoryCategory(QSqlQueryModel):
+
+    """A completion category that queries the SQL History store."""
+
+    def __init__(self, *, delete_func=None, parent=None):
+        """Create a new History completion category."""
+        super().__init__(parent=parent)
+        self.name = "History"
+
+        # replace ' in timestamp-format to avoid breaking the query
+        timefmt = ("strftime('{}', last_atime, 'unixepoch', 'localtime')"
+                   .format(config.get('completion', 'timestamp-format')
+                           .replace("'", "`")))
+
+        self._query = sql.Query(' '.join([
+            "SELECT url, title, {}".format(timefmt),
+            "FROM CompletionHistory",
+            # the incoming pattern will have literal % and _ escaped with '\'
+            # we need to tell sql to treat '\' as an escape character
+            "WHERE (url LIKE :pat escape '\\' or title LIKE :pat escape '\\')",
+            self._atime_expr(),
+            "ORDER BY last_atime DESC",
+        ]), forward_only=False)
+
+        # advertise that this model filters by URL and title
+        self.columns_to_filter = [0, 1]
+        self.delete_func = delete_func
+
+    def _atime_expr(self):
+        """If max_items is set, return an expression to limit the query."""
+        max_items = config.get('completion', 'web-history-max-items')
+        if max_items < 0:
+            return ''
+
+        min_atime = sql.Query(' '.join([
+            'SELECT min(last_atime) FROM',
+            '(SELECT last_atime FROM CompletionHistory',
+            'ORDER BY last_atime DESC LIMIT :limit)',
+        ])).run(limit=max_items).value()
+
+        return "AND last_atime >= {}".format(min_atime)
+
+    def set_pattern(self, pattern):
+        """Set the pattern used to filter results.
+
+        Args:
+            pattern: string pattern to filter by.
+        """
+        # escape to treat a user input % or _ as a literal, not a wildcard
+        pattern = pattern.replace('%', '\\%')
+        pattern = pattern.replace('_', '\\_')
+        # treat spaces as wildcards to match any of the typed words
+        pattern = re.sub(r' +', '%', pattern)
+        pattern = '%{}%'.format(pattern)
+        with debug.log_time('sql', 'Running completion query'):
+            self._query.run(pat=pattern)
+        self.setQuery(self._query)
+
+    def delete_cur_item(self, index):
+        """Delete the row at the given index."""
+        if not self.delete_func:
+            raise cmdexc.CommandError("Cannot delete this item.")
+        data = [self.data(index.sibling(index.row(), i))
+                for i in range(self.columnCount())]
+        self.delete_func(data)
+        # re-run query to reload updated table
+        with debug.log_time('sql', 'Re-running completion query post-delete'):
+            self._query.run()
+        self.setQuery(self._query)

qutebrowser/completion/models/instances.py

@@ -1,196 +0,0 @@
-# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
-
-# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Global instances of the completion models.
-
-Module attributes:
-    _instances: A dict of available completions.
-    INITIALIZERS: A {usertypes.Completion: callable} dict of functions to
-                  initialize completions.
-"""
-
-import functools
-
-from qutebrowser.completion.models import miscmodels, urlmodel, configmodel
-from qutebrowser.utils import objreg, usertypes, log, debug
-from qutebrowser.config import configdata, config
-
-
-_instances = {}
-
-
-def _init_command_completion():
-    """Initialize the command completion model."""
-    log.completion.debug("Initializing command completion.")
-    model = miscmodels.CommandCompletionModel()
-    _instances[usertypes.Completion.command] = model
-
-
-def _init_helptopic_completion():
-    """Initialize the helptopic completion model."""
-    log.completion.debug("Initializing helptopic completion.")
-    model = miscmodels.HelpCompletionModel()
-    _instances[usertypes.Completion.helptopic] = model
-
-
-def _init_url_completion():
-    """Initialize the URL completion model."""
-    log.completion.debug("Initializing URL completion.")
-    with debug.log_time(log.completion, 'URL completion init'):
-        model = urlmodel.UrlCompletionModel()
-        _instances[usertypes.Completion.url] = model
-
-
-def _init_tab_completion():
-    """Initialize the tab completion model."""
-    log.completion.debug("Initializing tab completion.")
-    with debug.log_time(log.completion, 'tab completion init'):
-        model = miscmodels.TabCompletionModel()
-        _instances[usertypes.Completion.tab] = model
-
-
-def _init_setting_completions():
-    """Initialize setting completion models."""
-    log.completion.debug("Initializing setting completion.")
-    _instances[usertypes.Completion.section] = (
-        configmodel.SettingSectionCompletionModel())
-    _instances[usertypes.Completion.option] = {}
-    _instances[usertypes.Completion.value] = {}
-    for sectname in configdata.DATA:
-        opt_model = configmodel.SettingOptionCompletionModel(sectname)
-        _instances[usertypes.Completion.option][sectname] = opt_model
-        _instances[usertypes.Completion.value][sectname] = {}
-        for opt in configdata.DATA[sectname]:
-            val_model = configmodel.SettingValueCompletionModel(sectname, opt)
-            _instances[usertypes.Completion.value][sectname][opt] = val_model
-
-
-def init_quickmark_completions():
-    """Initialize quickmark completion models."""
-    log.completion.debug("Initializing quickmark completion.")
-    try:
-        _instances[usertypes.Completion.quickmark_by_name].deleteLater()
-    except KeyError:
-        pass
-    model = miscmodels.QuickmarkCompletionModel()
-    _instances[usertypes.Completion.quickmark_by_name] = model
-
-
-def init_bookmark_completions():
-    """Initialize bookmark completion models."""
-    log.completion.debug("Initializing bookmark completion.")
-    try:
-        _instances[usertypes.Completion.bookmark_by_url].deleteLater()
-    except KeyError:
-        pass
-    model = miscmodels.BookmarkCompletionModel()
-    _instances[usertypes.Completion.bookmark_by_url] = model
-
-
-def init_session_completion():
-    """Initialize session completion model."""
-    log.completion.debug("Initializing session completion.")
-    try:
-        _instances[usertypes.Completion.sessions].deleteLater()
-    except KeyError:
-        pass
-    model = miscmodels.SessionCompletionModel()
-    _instances[usertypes.Completion.sessions] = model
-
-
-def _init_bind_completion():
-    """Initialize the command completion model."""
-    log.completion.debug("Initializing bind completion.")
-    model = miscmodels.BindCompletionModel()
-    _instances[usertypes.Completion.bind] = model
-
-
-INITIALIZERS = {
-    usertypes.Completion.command: _init_command_completion,
-    usertypes.Completion.helptopic: _init_helptopic_completion,
-    usertypes.Completion.url: _init_url_completion,
-    usertypes.Completion.tab: _init_tab_completion,
-    usertypes.Completion.section: _init_setting_completions,
-    usertypes.Completion.option: _init_setting_completions,
-    usertypes.Completion.value: _init_setting_completions,
-    usertypes.Completion.quickmark_by_name: init_quickmark_completions,
-    usertypes.Completion.bookmark_by_url: init_bookmark_completions,
-    usertypes.Completion.sessions: init_session_completion,
-    usertypes.Completion.bind: _init_bind_completion,
-}
-
-
-def get(completion):
-    """Get a certain completion. Initializes the completion if needed."""
-    try:
-        return _instances[completion]
-    except KeyError:
-        if completion in INITIALIZERS:
-            INITIALIZERS[completion]()
-            return _instances[completion]
-        else:
-            raise
-
-
-def update(completions):
-    """Update an already existing completion.
-
-    Args:
-        completions: An iterable of usertypes.Completions.
-    """
-    did_run = []
-    for completion in completions:
-        if completion in _instances:
-            func = INITIALIZERS[completion]
-            if func not in did_run:
-                func()
-                did_run.append(func)
-
-
-@config.change_filter('aliases', function=True)
-def _update_aliases():
-    """Update completions that include command aliases."""
-    update([usertypes.Completion.command])
-
-
-def init():
-    """Initialize completions. Note this only connects signals."""
-    quickmark_manager = objreg.get('quickmark-manager')
-    quickmark_manager.changed.connect(
-        functools.partial(update, [usertypes.Completion.quickmark_by_name]))
-
-    bookmark_manager = objreg.get('bookmark-manager')
-    bookmark_manager.changed.connect(
-        functools.partial(update, [usertypes.Completion.bookmark_by_url]))
-
-    session_manager = objreg.get('session-manager')
-    session_manager.update_completion.connect(
-        functools.partial(update, [usertypes.Completion.sessions]))
-
-    history = objreg.get('web-history')
-    history.async_read_done.connect(
-        functools.partial(update, [usertypes.Completion.url]))
-
-    keyconf = objreg.get('key-config')
-    keyconf.changed.connect(
-        functools.partial(update, [usertypes.Completion.command]))
-    keyconf.changed.connect(
-        functools.partial(update, [usertypes.Completion.bind]))
-
-    objreg.get('config').changed.connect(_update_aliases)

qutebrowser/completion/models/listcategory.py

@@ -0,0 +1,102 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Ryan Roden-Corrent (rcorre) <ryan@rcorre.net>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Completion category that uses a list of tuples as a data source."""
+
+import re
+
+from PyQt5.QtCore import Qt, QSortFilterProxyModel, QModelIndex, QRegExp
+from PyQt5.QtGui import QStandardItem, QStandardItemModel
+
+from qutebrowser.utils import qtutils
+from qutebrowser.commands import cmdexc
+
+
+class ListCategory(QSortFilterProxyModel):
+
+    """Expose a list of items as a category for the CompletionModel."""
+
+    def __init__(self, name, items, delete_func=None, parent=None):
+        super().__init__(parent)
+        self.name = name
+        self.srcmodel = QStandardItemModel(parent=self)
+        self._pattern = ''
+        # ListCategory filters all columns
+        self.columns_to_filter = [0, 1, 2]
+        self.setFilterKeyColumn(-1)
+        for item in items:
+            self.srcmodel.appendRow([QStandardItem(x) for x in item])
+        self.setSourceModel(self.srcmodel)
+        self.delete_func = delete_func
+
+    def set_pattern(self, val):
+        """Setter for pattern.
+
+        Args:
+            val: The value to set.
+        """
+        self._pattern = val
+        val = re.sub(r' +', r' ', val)  # See #1919
+        val = re.escape(val)
+        val = val.replace(r'\ ', '.*')
+        rx = QRegExp(val, Qt.CaseInsensitive)
+        self.setFilterRegExp(rx)
+        self.invalidate()
+        sortcol = 0
+        self.sort(sortcol)
+
+    def lessThan(self, lindex, rindex):
+        """Custom sorting implementation.
+
+        Prefers all items which start with self._pattern. Other than that, uses
+        normal Python string sorting.
+
+        Args:
+            lindex: The QModelIndex of the left item (*left* < right)
+            rindex: The QModelIndex of the right item (left < *right*)
+
+        Return:
+            True if left < right, else False
+        """
+        qtutils.ensure_valid(lindex)
+        qtutils.ensure_valid(rindex)
+
+        left = self.srcmodel.data(lindex)
+        right = self.srcmodel.data(rindex)
+
+        leftstart = left.startswith(self._pattern)
+        rightstart = right.startswith(self._pattern)
+
+        if leftstart and rightstart:
+            return left < right
+        elif leftstart:
+            return True
+        elif rightstart:
+            return False
+        else:
+            return left < right
+
+    def delete_cur_item(self, index):
+        """Delete the row at the given index."""
+        if not self.delete_func:
+            raise cmdexc.CommandError("Cannot delete this item.")
+        data = [self.data(index.sibling(index.row(), i))
+                for i in range(self.columnCount())]
+        self.delete_func(data)
+        self.removeRow(index.row(), QModelIndex())

qutebrowser/completion/models/miscmodels.py

@@ -17,60 +17,29 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Misc. CompletionModels."""
+"""Functions that return miscellaneous completion models."""
 
-from PyQt5.QtCore import Qt, QTimer, pyqtSlot
-
-from qutebrowser.browser import browsertab
 from qutebrowser.config import config, configdata
-from qutebrowser.utils import objreg, log, qtutils
+from qutebrowser.utils import objreg, log
 from qutebrowser.commands import cmdutils
-from qutebrowser.completion.models import base
+from qutebrowser.completion.models import completionmodel, listcategory
 
 
-class CommandCompletionModel(base.BaseCompletionModel):
+def command():
-
     """A CompletionModel filled with non-hidden commands and descriptions."""
-
+    model = completionmodel.CompletionModel(column_widths=(20, 60, 20))
-    # https://github.com/qutebrowser/qutebrowser/issues/545
+    cmdlist = _get_cmd_completions(include_aliases=True, include_hidden=False)
-    # pylint: disable=abstract-method
+    model.add_category(listcategory.ListCategory("Commands", cmdlist))
-
+    return model
-    COLUMN_WIDTHS = (20, 60, 20)
-
-    def __init__(self, parent=None):
-        super().__init__(parent)
-        cmdlist = _get_cmd_completions(include_aliases=True,
-                                       include_hidden=False)
-        cat = self.new_category("Commands")
-        for (name, desc, misc) in cmdlist:
-            self.new_item(cat, name, desc, misc)
 
 
-class HelpCompletionModel(base.BaseCompletionModel):
+def helptopic():
-
     """A CompletionModel filled with help topics."""
+    model = completionmodel.CompletionModel()
 
-    # https://github.com/qutebrowser/qutebrowser/issues/545
+    cmdlist = _get_cmd_completions(include_aliases=False, include_hidden=True,
-    # pylint: disable=abstract-method
+                                   prefix=':')
-
+    settings = []
-    COLUMN_WIDTHS = (20, 60, 20)
-
-    def __init__(self, parent=None):
-        super().__init__(parent)
-        self._init_commands()
-        self._init_settings()
-
-    def _init_commands(self):
-        """Fill completion with :command entries."""
-        cmdlist = _get_cmd_completions(include_aliases=False,
-                                       include_hidden=True, prefix=':')
-        cat = self.new_category("Commands")
-        for (name, desc, misc) in cmdlist:
-            self.new_item(cat, name, desc, misc)
-
-    def _init_settings(self):
-        """Fill completion with section->option entries."""
-        cat = self.new_category("Settings")
     for sectname, sectdata in configdata.DATA.items():
         for optname in sectdata:
             try:
@@ -82,187 +51,91 @@ class HelpCompletionModel(base.BaseCompletionModel):
             else:
                 desc = desc.splitlines()[0]
             name = '{}->{}'.format(sectname, optname)
-                self.new_item(cat, name, desc)
+            settings.append((name, desc))
+
+    model.add_category(listcategory.ListCategory("Commands", cmdlist))
+    model.add_category(listcategory.ListCategory("Settings", settings))
+    return model
 
 
-class QuickmarkCompletionModel(base.BaseCompletionModel):
+def quickmark():
-
     """A CompletionModel filled with all quickmarks."""
-
+    model = completionmodel.CompletionModel(column_widths=(30, 70, 0))
-    # https://github.com/qutebrowser/qutebrowser/issues/545
+    marks = objreg.get('quickmark-manager').marks.items()
-    # pylint: disable=abstract-method
+    model.add_category(listcategory.ListCategory('Quickmarks', marks))
-
+    return model
-    def __init__(self, parent=None):
-        super().__init__(parent)
-        cat = self.new_category("Quickmarks")
-        quickmarks = objreg.get('quickmark-manager').marks.items()
-        for qm_name, qm_url in quickmarks:
-            self.new_item(cat, qm_name, qm_url)
 
 
-class BookmarkCompletionModel(base.BaseCompletionModel):
+def bookmark():
-
     """A CompletionModel filled with all bookmarks."""
-
+    model = completionmodel.CompletionModel(column_widths=(30, 70, 0))
-    # https://github.com/qutebrowser/qutebrowser/issues/545
+    marks = objreg.get('bookmark-manager').marks.items()
-    # pylint: disable=abstract-method
+    model.add_category(listcategory.ListCategory('Bookmarks', marks))
-
+    return model
-    def __init__(self, parent=None):
-        super().__init__(parent)
-        cat = self.new_category("Bookmarks")
-        bookmarks = objreg.get('bookmark-manager').marks.items()
-        for bm_url, bm_title in bookmarks:
-            self.new_item(cat, bm_url, bm_title)
 
 
-class SessionCompletionModel(base.BaseCompletionModel):
+def session():
-
     """A CompletionModel filled with session names."""
-
+    model = completionmodel.CompletionModel()
-    # https://github.com/qutebrowser/qutebrowser/issues/545
-    # pylint: disable=abstract-method
-
-    def __init__(self, parent=None):
-        super().__init__(parent)
-        cat = self.new_category("Sessions")
     try:
-            for name in objreg.get('session-manager').list_sessions():
+        manager = objreg.get('session-manager')
-                if not name.startswith('_'):
+        sessions = ((name,) for name in manager.list_sessions()
-                    self.new_item(cat, name)
+                    if not name.startswith('_'))
+        model.add_category(listcategory.ListCategory("Sessions", sessions))
     except OSError:
         log.completion.exception("Failed to list sessions!")
+    return model
 
 
-class TabCompletionModel(base.BaseCompletionModel):
+def buffer():
-
     """A model to complete on open tabs across all windows.
 
     Used for switching the buffer command.
     """
-
+    def delete_buffer(data):
-    IDX_COLUMN = 0
+        """Close the selected tab."""
-    URL_COLUMN = 1
+        win_id, tab_index = data[0].split('/')
-    TEXT_COLUMN = 2
-
-    COLUMN_WIDTHS = (6, 40, 54)
-    DUMB_SORT = Qt.DescendingOrder
-
-    def __init__(self, parent=None):
-        super().__init__(parent)
-
-        self.columns_to_filter = [self.IDX_COLUMN, self.URL_COLUMN,
-                                  self.TEXT_COLUMN]
-
-        for win_id in objreg.window_registry:
-            tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                        window=win_id)
-            for i in range(tabbed_browser.count()):
-                tab = tabbed_browser.widget(i)
-                tab.url_changed.connect(self.rebuild)
-                tab.title_changed.connect(self.rebuild)
-                tab.shutting_down.connect(self.delayed_rebuild)
-            tabbed_browser.new_tab.connect(self.on_new_tab)
-            tabbed_browser.tabBar().tabMoved.connect(self.rebuild)
-        objreg.get("app").new_window.connect(self.on_new_window)
-        self.rebuild()
-
-    def on_new_window(self, window):
-        """Add hooks to new windows."""
-        window.tabbed_browser.new_tab.connect(self.on_new_tab)
-
-    @pyqtSlot(browsertab.AbstractTab)
-    def on_new_tab(self, tab):
-        """Add hooks to new tabs."""
-        tab.url_changed.connect(self.rebuild)
-        tab.title_changed.connect(self.rebuild)
-        tab.shutting_down.connect(self.delayed_rebuild)
-        self.rebuild()
-
-    @pyqtSlot()
-    def delayed_rebuild(self):
-        """Fire a rebuild indirectly so widgets get a chance to update."""
-        QTimer.singleShot(0, self.rebuild)
-
-    @pyqtSlot()
-    def rebuild(self):
-        """Rebuild completion model from current tabs.
-
-        Very lazy method of keeping the model up to date. We could connect to
-        signals for new tab, tab url/title changed, tab close, tab moved and
-        make sure we handled background loads too ... but iterating over a
-        few/few dozen/few hundred tabs doesn't take very long at all.
-        """
-        window_count = 0
-        for win_id in objreg.window_registry:
-            tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                        window=win_id)
-            if not tabbed_browser.shutting_down:
-                window_count += 1
-
-        if window_count < self.rowCount():
-            self.removeRows(window_count, self.rowCount() - window_count)
-
-        for i, win_id in enumerate(objreg.window_registry):
-            tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                        window=win_id)
-            if tabbed_browser.shutting_down:
-                continue
-            if i >= self.rowCount():
-                c = self.new_category("{}".format(win_id))
-            else:
-                c = self.item(i, 0)
-                c.setData("{}".format(win_id), Qt.DisplayRole)
-            if tabbed_browser.count() < c.rowCount():
-                c.removeRows(tabbed_browser.count(),
-                             c.rowCount() - tabbed_browser.count())
-            for idx in range(tabbed_browser.count()):
-                tab = tabbed_browser.widget(idx)
-                if idx >= c.rowCount():
-                    self.new_item(c, "{}/{}".format(win_id, idx + 1),
-                                  tab.url().toDisplayString(),
-                                  tabbed_browser.page_title(idx))
-                else:
-                    c.child(idx, 0).setData("{}/{}".format(win_id, idx + 1),
-                                            Qt.DisplayRole)
-                    c.child(idx, 1).setData(tab.url().toDisplayString(),
-                                            Qt.DisplayRole)
-                    c.child(idx, 2).setData(tabbed_browser.page_title(idx),
-                                            Qt.DisplayRole)
-
-    def delete_cur_item(self, completion):
-        """Delete the selected item.
-
-        Args:
-            completion: The Completion object to use.
-        """
-        index = completion.currentIndex()
-        qtutils.ensure_valid(index)
-        category = index.parent()
-        qtutils.ensure_valid(category)
-        index = category.child(index.row(), self.IDX_COLUMN)
-        win_id, tab_index = index.data().split('/')
-
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window=int(win_id))
         tabbed_browser.on_tab_close_requested(int(tab_index) - 1)
 
+    model = completionmodel.CompletionModel(column_widths=(6, 40, 54))
 
-class BindCompletionModel(base.BaseCompletionModel):
+    for win_id in objreg.window_registry:
+        tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                    window=win_id)
+        if tabbed_browser.shutting_down:
+            continue
+        tabs = []
+        for idx in range(tabbed_browser.count()):
+            tab = tabbed_browser.widget(idx)
+            tabs.append(("{}/{}".format(win_id, idx + 1),
+                         tab.url().toDisplayString(),
+                         tabbed_browser.page_title(idx)))
+        cat = listcategory.ListCategory("{}".format(win_id), tabs,
+            delete_func=delete_buffer)
+        model.add_category(cat)
 
-    """A CompletionModel filled with all bindable commands and descriptions."""
+    return model
 
-    # https://github.com/qutebrowser/qutebrowser/issues/545
-    # pylint: disable=abstract-method
 
-    COLUMN_WIDTHS = (20, 60, 20)
+def bind(key):
+    """A CompletionModel filled with all bindable commands and descriptions.
 
-    def __init__(self, parent=None):
+    Args:
-        super().__init__(parent)
+        key: the key being bound.
-        cmdlist = _get_cmd_completions(include_hidden=True,
+    """
-                                       include_aliases=True)
+    model = completionmodel.CompletionModel(column_widths=(20, 60, 20))
-        cat = self.new_category("Commands")
+    cmd_name = objreg.get('key-config').get_bindings_for('normal').get(key)
-        for (name, desc, misc) in cmdlist:
+
-            self.new_item(cat, name, desc, misc)
+    if cmd_name:
+        cmd = cmdutils.cmd_dict.get(cmd_name)
+        data = [(cmd_name, cmd.desc, key)]
+        model.add_category(listcategory.ListCategory("Current", data))
+
+    cmdlist = _get_cmd_completions(include_hidden=True, include_aliases=True)
+    model.add_category(listcategory.ListCategory("Commands", cmdlist))
+    return model
 
 
 def _get_cmd_completions(include_hidden, include_aliases, prefix=''):

qutebrowser/completion/models/sortfilter.py

@@ -1,191 +0,0 @@
-# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
-
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-"""A filtering/sorting base model for completions.
-
-Contains:
-    CompletionFilterModel -- A QSortFilterProxyModel subclass for completions.
-"""
-
-import re
-
-from PyQt5.QtCore import QSortFilterProxyModel, QModelIndex, Qt
-
-from qutebrowser.utils import log, qtutils, debug
-from qutebrowser.completion.models import base as completion
-
-
-class CompletionFilterModel(QSortFilterProxyModel):
-
-    """Subclass of QSortFilterProxyModel with custom sorting/filtering.
-
-    Attributes:
-        pattern: The pattern to filter with.
-        srcmodel: The current source model.
-                   Kept as attribute because calling `sourceModel` takes quite
-                   a long time for some reason.
-        _sort_order: The order to use for sorting if using dumb_sort.
-    """
-
-    def __init__(self, source, parent=None):
-        super().__init__(parent)
-        super().setSourceModel(source)
-        self.srcmodel = source
-        self.pattern = ''
-        self.pattern_re = None
-
-        dumb_sort = self.srcmodel.DUMB_SORT
-        if dumb_sort is None:
-            # pylint: disable=invalid-name
-            self.lessThan = self.intelligentLessThan
-            self._sort_order = Qt.AscendingOrder
-        else:
-            self.setSortRole(completion.Role.sort)
-            self._sort_order = dumb_sort
-
-    def set_pattern(self, val):
-        """Setter for pattern.
-
-        Invalidates the filter and re-sorts the model.
-
-        Args:
-            val: The value to set.
-        """
-        with debug.log_time(log.completion, 'Setting filter pattern'):
-            self.pattern = val
-            val = re.sub(r' +', r' ', val)  # See #1919
-            val = re.escape(val)
-            val = val.replace(r'\ ', '.*')
-            self.pattern_re = re.compile(val, re.IGNORECASE)
-            self.invalidate()
-            sortcol = 0
-            self.sort(sortcol)
-
-    def count(self):
-        """Get the count of non-toplevel items currently visible.
-
-        Note this only iterates one level deep, as we only need root items
-        (categories) and children (items) in our model.
-        """
-        count = 0
-        for i in range(self.rowCount()):
-            cat = self.index(i, 0)
-            qtutils.ensure_valid(cat)
-            count += self.rowCount(cat)
-        return count
-
-    def first_item(self):
-        """Return the first item in the model."""
-        for i in range(self.rowCount()):
-            cat = self.index(i, 0)
-            qtutils.ensure_valid(cat)
-            if cat.model().hasChildren(cat):
-                index = self.index(0, 0, cat)
-                qtutils.ensure_valid(index)
-                return index
-        return QModelIndex()
-
-    def last_item(self):
-        """Return the last item in the model."""
-        for i in range(self.rowCount() - 1, -1, -1):
-            cat = self.index(i, 0)
-            qtutils.ensure_valid(cat)
-            if cat.model().hasChildren(cat):
-                index = self.index(self.rowCount(cat) - 1, 0, cat)
-                qtutils.ensure_valid(index)
-                return index
-        return QModelIndex()
-
-    def setSourceModel(self, model):
-        """Override QSortFilterProxyModel's setSourceModel to clear pattern."""
-        log.completion.debug("Setting source model: {}".format(model))
-        self.set_pattern('')
-        super().setSourceModel(model)
-        self.srcmodel = model
-
-    def filterAcceptsRow(self, row, parent):
-        """Custom filter implementation.
-
-        Override QSortFilterProxyModel::filterAcceptsRow.
-
-        Args:
-            row: The row of the item.
-            parent: The parent item QModelIndex.
-
-        Return:
-            True if self.pattern is contained in item, or if it's a root item
-            (category). False in all other cases
-        """
-        if parent == QModelIndex() or not self.pattern:
-            return True
-
-        for col in self.srcmodel.columns_to_filter:
-            idx = self.srcmodel.index(row, col, parent)
-            if not idx.isValid():  # pragma: no cover
-                # this is a sanity check not hit by any test case
-                continue
-            data = self.srcmodel.data(idx)
-            if not data:
-                continue
-            elif self.pattern_re.search(data):
-                return True
-        return False
-
-    def intelligentLessThan(self, lindex, rindex):
-        """Custom sorting implementation.
-
-        Prefers all items which start with self.pattern. Other than that, uses
-        normal Python string sorting.
-
-        Args:
-            lindex: The QModelIndex of the left item (*left* < right)
-            rindex: The QModelIndex of the right item (left < *right*)
-
-        Return:
-            True if left < right, else False
-        """
-        qtutils.ensure_valid(lindex)
-        qtutils.ensure_valid(rindex)
-
-        left_sort = self.srcmodel.data(lindex, role=completion.Role.sort)
-        right_sort = self.srcmodel.data(rindex, role=completion.Role.sort)
-
-        if left_sort is not None and right_sort is not None:
-            return left_sort < right_sort
-
-        left = self.srcmodel.data(lindex)
-        right = self.srcmodel.data(rindex)
-
-        leftstart = left.startswith(self.pattern)
-        rightstart = right.startswith(self.pattern)
-
-        if leftstart and rightstart:
-            return left < right
-        elif leftstart:
-            return True
-        elif rightstart:
-            return False
-        else:
-            return left < right
-
-    def sort(self, column, order=None):
-        """Extend sort to respect self._sort_order if no order was given."""
-        if order is None:
-            order = self._sort_order
-        super().sort(column, order)

qutebrowser/completion/models/urlmodel.py

@@ -17,176 +17,54 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""CompletionModels for URLs."""
+"""Function to return the url completion model for the `open` command."""
 
-import datetime
+from qutebrowser.completion.models import (completionmodel, listcategory,
-
+                                           histcategory)
-from PyQt5.QtCore import pyqtSlot, Qt
+from qutebrowser.utils import log, objreg
-
-from qutebrowser.utils import objreg, utils, qtutils, log
-from qutebrowser.completion.models import base
-from qutebrowser.config import config
 
 
-class UrlCompletionModel(base.BaseCompletionModel):
+_URLCOL = 0
+_TEXTCOL = 1
 
+
+def _delete_history(data):
+    urlstr = data[_URLCOL]
+    log.completion.debug('Deleting history entry {}'.format(urlstr))
+    hist = objreg.get('web-history')
+    hist.delete_url(urlstr)
+
+
+def _delete_bookmark(data):
+    urlstr = data[_URLCOL]
+    log.completion.debug('Deleting bookmark {}'.format(urlstr))
+    bookmark_manager = objreg.get('bookmark-manager')
+    bookmark_manager.delete(urlstr)
+
+
+def _delete_quickmark(data):
+    name = data[_TEXTCOL]
+    quickmark_manager = objreg.get('quickmark-manager')
+    log.completion.debug('Deleting quickmark {}'.format(name))
+    quickmark_manager.delete(name)
+
+
+def url():
     """A model which combines bookmarks, quickmarks and web history URLs.
 
     Used for the `open` command.
     """
+    model = completionmodel.CompletionModel(column_widths=(40, 50, 10))
 
-    URL_COLUMN = 0
+    quickmarks = ((url, name) for (name, url)
-    TEXT_COLUMN = 1
+                  in objreg.get('quickmark-manager').marks.items())
-    TIME_COLUMN = 2
+    bookmarks = objreg.get('bookmark-manager').marks.items()
 
-    COLUMN_WIDTHS = (40, 50, 10)
+    model.add_category(listcategory.ListCategory(
-    DUMB_SORT = Qt.DescendingOrder
+        'Quickmarks', quickmarks, delete_func=_delete_quickmark))
+    model.add_category(listcategory.ListCategory(
+        'Bookmarks', bookmarks, delete_func=_delete_bookmark))
 
-    def __init__(self, parent=None):
+    hist_cat = histcategory.HistoryCategory(delete_func=_delete_history)
-        super().__init__(parent)
+    model.add_category(hist_cat)
-
+    return model
-        self.columns_to_filter = [self.URL_COLUMN, self.TEXT_COLUMN]
-
-        self._quickmark_cat = self.new_category("Quickmarks")
-        self._bookmark_cat = self.new_category("Bookmarks")
-        self._history_cat = self.new_category("History")
-
-        quickmark_manager = objreg.get('quickmark-manager')
-        quickmarks = quickmark_manager.marks.items()
-        for qm_name, qm_url in quickmarks:
-            self.new_item(self._quickmark_cat, qm_url, qm_name)
-        quickmark_manager.added.connect(
-            lambda name, url: self.new_item(self._quickmark_cat, url, name))
-        quickmark_manager.removed.connect(self.on_quickmark_removed)
-
-        bookmark_manager = objreg.get('bookmark-manager')
-        bookmarks = bookmark_manager.marks.items()
-        for bm_url, bm_title in bookmarks:
-            self.new_item(self._bookmark_cat, bm_url, bm_title)
-        bookmark_manager.added.connect(
-            lambda name, url: self.new_item(self._bookmark_cat, url, name))
-        bookmark_manager.removed.connect(self.on_bookmark_removed)
-
-        self._history = objreg.get('web-history')
-        self._max_history = config.get('completion', 'web-history-max-items')
-        history = utils.newest_slice(self._history, self._max_history)
-        for entry in history:
-            if not entry.redirect:
-                self._add_history_entry(entry)
-        self._history.add_completion_item.connect(self.on_history_item_added)
-        self._history.cleared.connect(self.on_history_cleared)
-
-        objreg.get('config').changed.connect(self.reformat_timestamps)
-
-    def _fmt_atime(self, atime):
-        """Format an atime to a human-readable string."""
-        fmt = config.get('completion', 'timestamp-format')
-        if fmt is None:
-            return ''
-        try:
-            dt = datetime.datetime.fromtimestamp(atime)
-        except (ValueError, OSError, OverflowError):
-            # Different errors which can occur for too large values...
-            log.misc.error("Got invalid timestamp {}!".format(atime))
-            return '(invalid)'
-        else:
-            return dt.strftime(fmt)
-
-    def _remove_oldest_history(self):
-        """Remove the oldest history entry."""
-        self._history_cat.removeRow(0)
-
-    def _add_history_entry(self, entry):
-        """Add a new history entry to the completion."""
-        self.new_item(self._history_cat, entry.url.toDisplayString(),
-                      entry.title,
-                      self._fmt_atime(entry.atime), sort=int(entry.atime),
-                      userdata=entry.url)
-
-        if (self._max_history != -1 and
-                self._history_cat.rowCount() > self._max_history):
-            self._remove_oldest_history()
-
-    @config.change_filter('completion', 'timestamp-format')
-    def reformat_timestamps(self):
-        """Reformat the timestamps if the config option was changed."""
-        for i in range(self._history_cat.rowCount()):
-            url_item = self._history_cat.child(i, self.URL_COLUMN)
-            atime_item = self._history_cat.child(i, self.TIME_COLUMN)
-            atime = url_item.data(base.Role.sort)
-            atime_item.setText(self._fmt_atime(atime))
-
-    @pyqtSlot(object)
-    def on_history_item_added(self, entry):
-        """Slot called when a new history item was added."""
-        for i in range(self._history_cat.rowCount()):
-            url_item = self._history_cat.child(i, self.URL_COLUMN)
-            atime_item = self._history_cat.child(i, self.TIME_COLUMN)
-            title_item = self._history_cat.child(i, self.TEXT_COLUMN)
-            url = url_item.data(base.Role.userdata)
-            if url == entry.url:
-                atime_item.setText(self._fmt_atime(entry.atime))
-                title_item.setText(entry.title)
-                url_item.setData(int(entry.atime), base.Role.sort)
-                break
-        else:
-            self._add_history_entry(entry)
-
-    @pyqtSlot()
-    def on_history_cleared(self):
-        self._history_cat.removeRows(0, self._history_cat.rowCount())
-
-    def _remove_item(self, data, category, column):
-        """Helper function for on_quickmark_removed and on_bookmark_removed.
-
-        Args:
-            data: The item to search for.
-            category: The category to search in.
-            column: The column to use for matching.
-        """
-        for i in range(category.rowCount()):
-            item = category.child(i, column)
-            if item.data(Qt.DisplayRole) == data:
-                category.removeRow(i)
-                break
-
-    @pyqtSlot(str)
-    def on_quickmark_removed(self, name):
-        """Called when a quickmark has been removed by the user.
-
-        Args:
-            name: The name of the quickmark which has been removed.
-        """
-        self._remove_item(name, self._quickmark_cat, self.TEXT_COLUMN)
-
-    @pyqtSlot(str)
-    def on_bookmark_removed(self, url):
-        """Called when a bookmark has been removed by the user.
-
-        Args:
-            url: The url of the bookmark which has been removed.
-        """
-        self._remove_item(url, self._bookmark_cat, self.URL_COLUMN)
-
-    def delete_cur_item(self, completion):
-        """Delete the selected item.
-
-        Args:
-            completion: The Completion object to use.
-        """
-        index = completion.currentIndex()
-        qtutils.ensure_valid(index)
-        category = index.parent()
-        index = category.child(index.row(), self.URL_COLUMN)
-        url = index.data()
-        qtutils.ensure_valid(category)
-
-        if category.data() == 'Bookmarks':
-            bookmark_manager = objreg.get('bookmark-manager')
-            bookmark_manager.delete(url)
-        elif category.data() == 'Quickmarks':
-            quickmark_manager = objreg.get('quickmark-manager')
-            sibling = index.sibling(index.row(), self.TEXT_COLUMN)
-            qtutils.ensure_valid(sibling)
-            name = sibling.data()
-            quickmark_manager.delete(name)

qutebrowser/config/config.py

@@ -38,13 +38,12 @@ from PyQt5.QtCore import pyqtSignal, QObject, QUrl, QSettings
 from PyQt5.QtGui import QColor
 
 from qutebrowser.config import configdata, configexc, textwrapper
-from qutebrowser.config.parsers import keyconf
 from qutebrowser.config.parsers import ini
 from qutebrowser.commands import cmdexc, cmdutils
 from qutebrowser.utils import (message, objreg, utils, standarddir, log,
                                qtutils, error, usertypes)
 from qutebrowser.misc import objects
-from qutebrowser.utils.usertypes import Completion
+from qutebrowser.completion.models import configmodel
 
 
 UNSET = object()
@@ -175,37 +174,6 @@ def _init_main_config(parent=None):
                     return
 
 
-def _init_key_config(parent):
-    """Initialize the key config.
-
-    Args:
-        parent: The parent to use for the KeyConfigParser.
-    """
-    args = objreg.get('args')
-    try:
-        key_config = keyconf.KeyConfigParser(standarddir.config(), 'keys.conf',
-                                             args.relaxed_config,
-                                             parent=parent)
-    except (keyconf.KeyConfigError, cmdexc.CommandError,
-            UnicodeDecodeError) as e:
-        log.init.exception(e)
-        errstr = "Error while reading key config:\n"
-        if e.lineno is not None:
-            errstr += "In line {}: ".format(e.lineno)
-        error.handle_fatal_exc(e, args, "Error while reading key config!",
-                               pre_text=errstr)
-        # We didn't really initialize much so far, so we just quit hard.
-        sys.exit(usertypes.Exit.err_key_config)
-    else:
-        objreg.register('key-config', key_config)
-        save_manager = objreg.get('save-manager')
-        filename = os.path.join(standarddir.config(), 'keys.conf')
-        save_manager.add_saveable(
-            'key-config', key_config.save, key_config.config_dirty,
-            config_opt=('general', 'auto-save-config'), filename=filename,
-            dirty=key_config.is_dirty)
-
-
 def _init_misc():
     """Initialize misc. config-related files."""
     save_manager = objreg.get('save-manager')
@@ -249,7 +217,6 @@ def init(parent=None):
         parent: The parent to pass to QObjects which get initialized.
     """
     _init_main_config(parent)
-    _init_key_config(parent)
     _init_misc()
 
 
@@ -794,9 +761,9 @@ class ConfigManager(QObject):
                 e.__class__.__name__, e))
 
     @cmdutils.register(name='set', instance='config', star_args_optional=True)
-    @cmdutils.argument('section_', completion=Completion.section)
+    @cmdutils.argument('section_', completion=configmodel.section)
-    @cmdutils.argument('option', completion=Completion.option)
+    @cmdutils.argument('option', completion=configmodel.option)
-    @cmdutils.argument('values', completion=Completion.value)
+    @cmdutils.argument('values', completion=configmodel.value)
     @cmdutils.argument('win_id', win_id=True)
     def set_command(self, win_id, section_=None, option=None, *values,
                     temp=False, print_=False):

qutebrowser/config/configdata.py

@@ -503,7 +503,7 @@ def data(readonly=False):
              "0: no history / -1: unlimited"),
 
             ('web-history-max-items',
-             SettingValue(typ.Int(minval=-1), '1000'),
+             SettingValue(typ.Int(minval=-1), '-1'),
              "How many URLs to show in the web history.\n\n"
              "0: no history / -1: unlimited"),
 

qutebrowser/config/parsers/keyconf.py

@@ -22,12 +22,44 @@
 import collections
 import os.path
 import itertools
+import sys
 
 from PyQt5.QtCore import pyqtSignal, QObject
 
 from qutebrowser.config import configdata, textwrapper
 from qutebrowser.commands import cmdutils, cmdexc
-from qutebrowser.utils import log, utils, qtutils, message, usertypes
+from qutebrowser.utils import (log, utils, qtutils, message, usertypes, objreg,
+                               standarddir, error)
+from qutebrowser.completion.models import miscmodels
+
+
+def init(parent=None):
+    """Read and save keybindings.
+
+    Args:
+        parent: The parent to use for the KeyConfigParser.
+    """
+    args = objreg.get('args')
+    try:
+        key_config = KeyConfigParser(standarddir.config(), 'keys.conf',
+                                     args.relaxed_config, parent=parent)
+    except (KeyConfigError, UnicodeDecodeError) as e:
+        log.init.exception(e)
+        errstr = "Error while reading key config:\n"
+        if e.lineno is not None:
+            errstr += "In line {}: ".format(e.lineno)
+        error.handle_fatal_exc(e, args, "Error while reading key config!",
+                               pre_text=errstr)
+        # We didn't really initialize much so far, so we just quit hard.
+        sys.exit(usertypes.Exit.err_key_config)
+    else:
+        objreg.register('key-config', key_config)
+        save_manager = objreg.get('save-manager')
+        filename = os.path.join(standarddir.config(), 'keys.conf')
+        save_manager.add_saveable(
+            'key-config', key_config.save, key_config.config_dirty,
+            config_opt=('general', 'auto-save-config'), filename=filename,
+            dirty=key_config.is_dirty)
 
 
 class KeyConfigError(Exception):
@@ -153,7 +185,7 @@ class KeyConfigParser(QObject):
 
     @cmdutils.register(instance='key-config', maxsplit=1, no_cmd_split=True,
                        no_replace_variables=True)
-    @cmdutils.argument('command', completion=usertypes.Completion.bind)
+    @cmdutils.argument('command', completion=miscmodels.bind)
     def bind(self, key, command=None, *, mode='normal', force=False):
         """Bind a key to a command.
 

qutebrowser/javascript/history.js

@@ -23,8 +23,12 @@ window.loadHistory = (function() {
     // Date of last seen item.
     var lastItemDate = null;
 
-    // The time to load next.
+    // Each request for new items includes the time of the last item and an
+    // offset. The offset is equal to the number of items from the previous
+    // request that had time=nextTime, and causes the next request to skip
+    // those items to avoid duplicates.
     var nextTime = null;
+    var nextOffset = 0;
 
     // The URL to fetch data from.
     var DATA_URL = "qute://history/data";
@@ -157,23 +161,28 @@ window.loadHistory = (function() {
             return;
         }
 
-        for (var i = 0, len = history.length - 1; i < len; i++) {
+        if (history.length === 0) {
-            var item = history[i];
-            var currentItemDate = new Date(item.time);
-            getSessionNode(currentItemDate).appendChild(makeHistoryRow(
-                item.url, item.title, currentItemDate.toLocaleTimeString()
-            ));
-            lastItemDate = currentItemDate;
-        }
-
-        var next = history[history.length - 1].next;
-        if (next === -1) {
             // Reached end of history
             window.onscroll = null;
             EOF_MESSAGE.style.display = "block";
             LOAD_LINK.style.display = "none";
-        } else {
+            return;
-            nextTime = next;
+        }
+
+        nextTime = history[history.length - 1].time;
+        nextOffset = 0;
+
+        for (var i = 0, len = history.length; i < len; i++) {
+            var item = history[i];
+            // python's time.time returns seconds, but js Date expects ms
+            var currentItemDate = new Date(item.time * 1000);
+            getSessionNode(currentItemDate).appendChild(makeHistoryRow(
+                item.url, item.title, currentItemDate.toLocaleTimeString()
+            ));
+            lastItemDate = currentItemDate;
+            if (item.time === nextTime) {
+                nextOffset++;
+            }
         }
     }
 
@@ -182,10 +191,11 @@ window.loadHistory = (function() {
      * @return {void}
      */
     function loadHistory() {
+        var url = DATA_URL.concat("?offset=", nextOffset.toString());
         if (nextTime === null) {
-            getJSON(DATA_URL, receiveHistory);
+            getJSON(url, receiveHistory);
         } else {
-            var url = DATA_URL.concat("?start_time=", nextTime.toString());
+            url = url.concat("&start_time=", nextTime.toString());
             getJSON(url, receiveHistory);
         }
     }

qutebrowser/misc/earlyinit.py

@@ -338,6 +338,7 @@ def check_libraries(backend):
                                  "or Install via pip.",
                          pip="PyYAML"),
         'PyQt5.QtQml': _missing_str("PyQt5.QtQml"),
+        'PyQt5.QtSql': _missing_str("PyQt5.QtSql"),
     }
     if backend == 'webengine':
         modules['PyQt5.QtWebEngineWidgets'] = _missing_str("QtWebEngine",

qutebrowser/misc/lineparser.py

@@ -21,7 +21,6 @@
 
 import os
 import os.path
-import itertools
 import contextlib
 
 from PyQt5.QtCore import pyqtSlot, pyqtSignal, QObject
@@ -96,7 +95,7 @@ class BaseLineParser(QObject):
         """
         assert self._configfile is not None
         if self._opened:
-            raise IOError("Refusing to double-open AppendLineParser.")
+            raise IOError("Refusing to double-open LineParser.")
         self._opened = True
         try:
             if self._binary:
@@ -133,73 +132,6 @@ class BaseLineParser(QObject):
         raise NotImplementedError
 
 
-class AppendLineParser(BaseLineParser):
-
-    """LineParser which reads lazily and appends data to existing one.
-
-    Attributes:
-        _new_data: The data which was added in this session.
-    """
-
-    def __init__(self, configdir, fname, *, parent=None):
-        super().__init__(configdir, fname, binary=False, parent=parent)
-        self.new_data = []
-        self._fileobj = None
-
-    def __iter__(self):
-        if self._fileobj is None:
-            raise ValueError("Iterating without open() being called!")
-        file_iter = (line.rstrip('\n') for line in self._fileobj)
-        return itertools.chain(file_iter, iter(self.new_data))
-
-    @contextlib.contextmanager
-    def open(self):
-        """Open the on-disk history file. Needed for __iter__."""
-        try:
-            with self._open('r') as f:
-                self._fileobj = f
-                yield
-        except FileNotFoundError:
-            self._fileobj = []
-            yield
-        finally:
-            self._fileobj = None
-
-    def get_recent(self, count=4096):
-        """Get the last count bytes from the underlying file."""
-        with self._open('r') as f:
-            f.seek(0, os.SEEK_END)
-            size = f.tell()
-            try:
-                if size - count > 0:
-                    offset = size - count
-                else:
-                    offset = 0
-                f.seek(offset)
-                data = f.readlines()
-            finally:
-                f.seek(0, os.SEEK_END)
-        return data
-
-    def save(self):
-        do_save = self._prepare_save()
-        if not do_save:
-            return
-        with self._open('a') as f:
-            self._write(f, self.new_data)
-        self.new_data = []
-        self._after_save()
-
-    def clear(self):
-        do_save = self._prepare_save()
-        if not do_save:
-            return
-        with self._open('w'):
-            pass
-        self.new_data = []
-        self._after_save()
-
-
 class LineParser(BaseLineParser):
 
     """Parser for configuration files which are simply line-based.
@@ -240,7 +172,7 @@ class LineParser(BaseLineParser):
     def save(self):
         """Save the config file."""
         if self._opened:
-            raise IOError("Refusing to double-open AppendLineParser.")
+            raise IOError("Refusing to double-open LineParser.")
         do_save = self._prepare_save()
         if not do_save:
             return

qutebrowser/misc/sessions.py

@@ -31,10 +31,11 @@ try:
 except ImportError:  # pragma: no cover
     from yaml import SafeLoader as YamlLoader, SafeDumper as YamlDumper
 
-from qutebrowser.utils import (standarddir, objreg, qtutils, log, usertypes,
+from qutebrowser.utils import (standarddir, objreg, qtutils, log, message,
-                               message, utils)
+                               utils)
 from qutebrowser.commands import cmdexc, cmdutils
 from qutebrowser.config import config
+from qutebrowser.completion.models import miscmodels
 
 
 default = object()  # Sentinel value
@@ -436,7 +437,7 @@ class SessionManager(QObject):
         return sessions
 
     @cmdutils.register(instance='session-manager')
-    @cmdutils.argument('name', completion=usertypes.Completion.sessions)
+    @cmdutils.argument('name', completion=miscmodels.session)
     def session_load(self, name, clear=False, temp=False, force=False):
         """Load a session.
 
@@ -464,7 +465,7 @@ class SessionManager(QObject):
                     win.close()
 
     @cmdutils.register(name=['session-save', 'w'], instance='session-manager')
-    @cmdutils.argument('name', completion=usertypes.Completion.sessions)
+    @cmdutils.argument('name', completion=miscmodels.session)
     @cmdutils.argument('win_id', win_id=True)
     @cmdutils.argument('with_private', flag='p')
     def session_save(self, name: str = default, current=False, quiet=False,
@@ -503,7 +504,7 @@ class SessionManager(QObject):
                 message.info("Saved session {}.".format(name))
 
     @cmdutils.register(instance='session-manager')
-    @cmdutils.argument('name', completion=usertypes.Completion.sessions)
+    @cmdutils.argument('name', completion=miscmodels.session)
     def session_delete(self, name, force=False):
         """Delete a session.
 

qutebrowser/misc/sql.py

@@ -0,0 +1,256 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) <ryan@rcorre.net>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Provides access to an in-memory sqlite database."""
+
+import collections
+
+from PyQt5.QtCore import QObject, pyqtSignal
+from PyQt5.QtSql import QSqlDatabase, QSqlQuery
+
+from qutebrowser.utils import log
+
+
+class SqlException(Exception):
+
+    """Raised on an error interacting with the SQL database."""
+
+    pass
+
+
+def init(db_path):
+    """Initialize the SQL database connection."""
+    database = QSqlDatabase.addDatabase('QSQLITE')
+    if not database.isValid():
+        raise SqlException('Failed to add database. '
+            'Are sqlite and Qt sqlite support installed?')
+    database.setDatabaseName(db_path)
+    if not database.open():
+        raise SqlException("Failed to open sqlite database at {}: {}"
+                           .format(db_path, database.lastError().text()))
+
+
+def close():
+    """Close the SQL connection."""
+    QSqlDatabase.removeDatabase(QSqlDatabase.database().connectionName())
+
+
+def version():
+    """Return the sqlite version string."""
+    try:
+        if not QSqlDatabase.database().isOpen():
+            init(':memory:')
+            ver = Query("select sqlite_version()").run().value()
+            close()
+            return ver
+        return Query("select sqlite_version()").run().value()
+    except SqlException as e:
+        return 'UNAVAILABLE ({})'.format(e)
+
+
+class Query(QSqlQuery):
+
+    """A prepared SQL Query."""
+
+    def __init__(self, querystr, forward_only=True):
+        """Prepare a new sql query.
+
+        Args:
+            querystr: String to prepare query from.
+            forward_only: Optimization for queries that will only step forward.
+                          Must be false for completion queries.
+        """
+        super().__init__(QSqlDatabase.database())
+        log.sql.debug('Preparing SQL query: "{}"'.format(querystr))
+        if not self.prepare(querystr):
+            raise SqlException('Failed to prepare query "{}": "{}"'.format(
+                querystr, self.lastError().text()))
+        self.setForwardOnly(forward_only)
+
+    def __iter__(self):
+        if not self.isActive():
+            raise SqlException("Cannot iterate inactive query")
+        rec = self.record()
+        fields = [rec.fieldName(i) for i in range(rec.count())]
+        rowtype = collections.namedtuple('ResultRow', fields)
+
+        while self.next():
+            rec = self.record()
+            yield rowtype(*[rec.value(i) for i in range(rec.count())])
+
+    def run(self, **values):
+        """Execute the prepared query."""
+        log.sql.debug('Running SQL query: "{}"'.format(self.lastQuery()))
+        for key, val in values.items():
+            self.bindValue(':{}'.format(key), val)
+        log.sql.debug('query bindings: {}'.format(self.boundValues()))
+        if not self.exec_():
+            raise SqlException('Failed to exec query "{}": "{}"'.format(
+                               self.lastQuery(), self.lastError().text()))
+        return self
+
+    def value(self):
+        """Return the result of a single-value query (e.g. an EXISTS)."""
+        if not self.next():
+            raise SqlException("No result for single-result query")
+        return self.record().value(0)
+
+
+class SqlTable(QObject):
+
+    """Interface to a sql table.
+
+    Attributes:
+        _name: Name of the SQL table this wraps.
+
+    Signals:
+        changed: Emitted when the table is modified.
+    """
+
+    changed = pyqtSignal()
+
+    def __init__(self, name, fields, constraints=None, parent=None):
+        """Create a new table in the sql database.
+
+        Raises SqlException if the table already exists.
+
+        Args:
+            name: Name of the table.
+            fields: A list of field names.
+            constraints: A dict mapping field names to constraint strings.
+        """
+        super().__init__(parent)
+        self._name = name
+
+        constraints = constraints or {}
+        column_defs = ['{} {}'.format(field, constraints.get(field, ''))
+                       for field in fields]
+        q = Query("CREATE TABLE IF NOT EXISTS {name} ({column_defs})"
+                  .format(name=name, column_defs=', '.join(column_defs)))
+
+        q.run()
+
+    def create_index(self, name, field):
+        """Create an index over this table.
+
+        Args:
+            name: Name of the index, should be unique.
+            field: Name of the field to index.
+        """
+        q = Query("CREATE INDEX IF NOT EXISTS {name} ON {table} ({field})"
+                  .format(name=name, table=self._name, field=field))
+        q.run()
+
+    def __iter__(self):
+        """Iterate rows in the table."""
+        q = Query("SELECT * FROM {table}".format(table=self._name))
+        q.run()
+        return iter(q)
+
+    def contains_query(self, field):
+        """Return a prepared query that checks for the existence of an item.
+
+        Args:
+            field: Field to match.
+        """
+        return Query(
+            "SELECT EXISTS(SELECT * FROM {table} WHERE {field} = :val)"
+            .format(table=self._name, field=field))
+
+    def __len__(self):
+        """Return the count of rows in the table."""
+        q = Query("SELECT count(*) FROM {table}".format(table=self._name))
+        q.run()
+        return q.value()
+
+    def delete(self, field, value):
+        """Remove all rows for which `field` equals `value`.
+
+        Args:
+            field: Field to use as the key.
+            value: Key value to delete.
+
+        Return:
+            The number of rows deleted.
+        """
+        q = Query("DELETE FROM {table} where {field} = :val"
+                  .format(table=self._name, field=field))
+        q.run(val=value)
+        if not q.numRowsAffected():
+            raise KeyError('No row with {} = "{}"'.format(field, value))
+        self.changed.emit()
+
+    def _insert_query(self, values, replace):
+        params = ', '.join(':{}'.format(key) for key in values)
+        verb = "REPLACE" if replace else "INSERT"
+        return Query("{verb} INTO {table} ({columns}) values({params})".format(
+            verb=verb, table=self._name, columns=', '.join(values),
+            params=params))
+
+    def insert(self, values, replace=False):
+        """Append a row to the table.
+
+        Args:
+            values: A dict with a value to insert for each field name.
+            replace: If set, replace existing values.
+        """
+        q = self._insert_query(values, replace)
+        q.run(**values)
+        self.changed.emit()
+
+    def insert_batch(self, values, replace=False):
+        """Performantly append multiple rows to the table.
+
+        Args:
+            values: A dict with a list of values to insert for each field name.
+            replace: If true, overwrite rows with a primary key match.
+        """
+        q = self._insert_query(values, replace)
+        for key, val in values.items():
+            q.bindValue(':{}'.format(key), val)
+
+        db = QSqlDatabase.database()
+        db.transaction()
+        if not q.execBatch():
+            raise SqlException('Failed to exec query "{}": "{}"'.format(
+                               q.lastQuery(), q.lastError().text()))
+        db.commit()
+        self.changed.emit()
+
+    def delete_all(self):
+        """Remove all rows from the table."""
+        Query("DELETE FROM {table}".format(table=self._name)).run()
+        self.changed.emit()
+
+    def select(self, sort_by, sort_order, limit=-1):
+        """Prepare, run, and return a select statement on this table.
+
+        Args:
+            sort_by: name of column to sort by.
+            sort_order: 'asc' or 'desc'.
+            limit: max number of rows in result, defaults to -1 (unlimited).
+
+        Return: A prepared and executed select query.
+        """
+        q = Query("SELECT * FROM {table} ORDER BY {sort_by} {sort_order} "
+                  "LIMIT :limit"
+                  .format(table=self._name, sort_by=sort_by,
+                          sort_order=sort_order))
+        q.run(limit=limit)
+        return q

qutebrowser/misc/utilcmds.py

@@ -170,8 +170,15 @@ def debug_cache_stats():
     """Print LRU cache stats."""
     config_info = objreg.get('config').get.cache_info()
     style_info = style.get_stylesheet.cache_info()
+    try:
+        from PyQt5.QtWebKit import QWebHistoryInterface
+        interface = QWebHistoryInterface.defaultInterface()
+        history_info = interface.historyContains.cache_info()
+    except ImportError:
+        history_info = None
     log.misc.debug('config: {}'.format(config_info))
     log.misc.debug('style: {}'.format(style_info))
+    log.misc.debug('history: {}'.format(history_info))
 
 
 @cmdutils.register(debug=True)

qutebrowser/utils/log.py

@@ -94,7 +94,7 @@ LOGGER_NAMES = [
     'commands', 'signals', 'downloads',
     'js', 'qt', 'rfc6266', 'ipc', 'shlexer',
     'save', 'message', 'config', 'sessions',
-    'webelem', 'prompt', 'network'
+    'webelem', 'prompt', 'network', 'sql'
 ]
 
 
@@ -141,6 +141,7 @@ sessions = logging.getLogger('sessions')
 webelem = logging.getLogger('webelem')
 prompt = logging.getLogger('prompt')
 network = logging.getLogger('network')
+sql = logging.getLogger('sql')
 
 
 ram_handler = None

qutebrowser/utils/usertypes.py

@@ -236,13 +236,6 @@ KeyMode = enum('KeyMode', ['normal', 'hint', 'command', 'yesno', 'prompt',
                            'jump_mark', 'record_macro', 'run_macro'])
 
 
-# Available command completions
-Completion = enum('Completion', ['command', 'section', 'option', 'value',
-                                 'helptopic', 'quickmark_by_name',
-                                 'bookmark_by_url', 'url', 'tab', 'sessions',
-                                 'bind'])
-
-
 # Exit statuses for errors. Needs to be an int for sys.exit.
 Exit = enum('Exit', ['ok', 'reserved', 'exception', 'err_ipc', 'err_init',
                      'err_config', 'err_key_config'], is_int=True, start=0)

qutebrowser/utils/utils.py

@@ -28,7 +28,6 @@ import os.path
 import collections
 import functools
 import contextlib
-import itertools
 import socket
 import shlex
 
@@ -737,25 +736,6 @@ def sanitize_filename(name, replacement='_'):
     return name
 
 
-def newest_slice(iterable, count):
-    """Get an iterable for the n newest items of the given iterable.
-
-    Args:
-        count: How many elements to get.
-               0: get no items:
-               n: get the n newest items
-              -1: get all items
-    """
-    if count < -1:
-        raise ValueError("count can't be smaller than -1!")
-    elif count == 0:
-        return []
-    elif count == -1 or len(iterable) < count:
-        return iterable
-    else:
-        return itertools.islice(iterable, len(iterable) - count, len(iterable))
-
-
 def set_clipboard(data, selection=False):
     """Set the clipboard to some given data."""
     if selection and not supports_selection():

qutebrowser/utils/version.py

@@ -45,7 +45,7 @@ except ImportError:  # pragma: no cover
 
 import qutebrowser
 from qutebrowser.utils import log, utils, standarddir, usertypes, qtutils
-from qutebrowser.misc import objects, earlyinit
+from qutebrowser.misc import objects, earlyinit, sql
 from qutebrowser.browser import pdfjs
 
 
@@ -328,6 +328,7 @@ def version():
 
     lines += [
         'pdf.js: {}'.format(_pdfjs_version()),
+        'sqlite: {}'.format(sql.version()),
         'QtNetwork SSL: {}\n'.format(QSslSocket.sslLibraryVersionString()
                                      if QSslSocket.supportsSsl() else 'no'),
     ]

scripts/dev/check_coverage.py

@@ -51,9 +51,9 @@ PERFECT_FILES = [
         'browser/webkit/cache.py'),
     ('tests/unit/browser/webkit/test_cookies.py',
         'browser/webkit/cookies.py'),
-    ('tests/unit/browser/webkit/test_history.py',
+    ('tests/unit/browser/test_history.py',
         'browser/history.py'),
-    ('tests/unit/browser/webkit/test_history.py',
+    ('tests/unit/browser/test_history.py',
         'browser/webkit/webkithistory.py'),
     ('tests/unit/browser/webkit/http/test_http.py',
         'browser/webkit/http.py'),
@@ -157,9 +157,11 @@ PERFECT_FILES = [
         'utils/javascript.py'),
 
     ('tests/unit/completion/test_models.py',
-        'completion/models/base.py'),
+        'completion/models/urlmodel.py'),
-    ('tests/unit/completion/test_sortfilter.py',
+    ('tests/unit/completion/test_histcategory.py',
-        'completion/models/sortfilter.py'),
+        'completion/models/histcategory.py'),
+    ('tests/unit/completion/test_listcategory.py',
+        'completion/models/listcategory.py'),
 
 ]
 

scripts/dev/ci/travis_install.sh

@@ -109,7 +109,7 @@ elif [[ $TRAVIS_OS_NAME == osx ]]; then
     exit 0
 fi
 
-pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit"
+pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit python3-pyqt5.qtsql libqt5sql5-sqlite"
 
 pip_install pip
 pip_install -r misc/requirements/requirements-tox.txt

tests/end2end/features/completion.feature

@@ -32,23 +32,23 @@ Feature: Using completion
 
     Scenario: Using command completion
         When I run :set-cmd-text :
-        Then the completion model should be CommandCompletionModel
+        Then the completion model should be command
 
     Scenario: Using help completion
         When I run :set-cmd-text -s :help
-        Then the completion model should be HelpCompletionModel
+        Then the completion model should be helptopic
 
     Scenario: Using quickmark completion
         When I run :set-cmd-text -s :quickmark-load
-        Then the completion model should be QuickmarkCompletionModel
+        Then the completion model should be quickmark
 
     Scenario: Using bookmark completion
         When I run :set-cmd-text -s :bookmark-load
-        Then the completion model should be BookmarkCompletionModel
+        Then the completion model should be bookmark
 
     Scenario: Using bind completion
         When I run :set-cmd-text -s :bind X
-        Then the completion model should be BindCompletionModel
+        Then the completion model should be bind
 
     Scenario: Using session completion
         Given I open data/hello.txt
@@ -62,37 +62,11 @@ Feature: Using completion
 
     Scenario: Using option completion
         When I run :set-cmd-text -s :set colors
-        Then the completion model should be SettingOptionCompletionModel
+        Then the completion model should be option
 
     Scenario: Using value completion
         When I run :set-cmd-text -s :set colors statusbar.bg
-        Then the completion model should be SettingValueCompletionModel
+        Then the completion model should be value
-
-    Scenario: Updating the completion in realtime
-        Given I have a fresh instance
-        And I set completion -> quick-complete to false
-        When I open data/hello.txt
-        And I run :set-cmd-text -s :buffer
-        And I run :completion-item-focus next
-        And I open data/hello2.txt in a new background tab
-        And I run :completion-item-focus next
-        And I open data/hello3.txt in a new background tab
-        And I run :completion-item-focus next
-        And I run :command-accept
-        Then the following tabs should be open:
-            - data/hello.txt
-            - data/hello2.txt
-            - data/hello3.txt (active)
-
-    Scenario: Updating the value completion in realtime
-        Given I set colors -> statusbar.bg to green
-        When I run :set-cmd-text -s :set colors statusbar.bg
-        And I set colors -> statusbar.bg to yellow
-        And I run :completion-item-focus next
-        And I run :completion-item-focus next
-        And I set colors -> statusbar.bg to red
-        And I run :command-accept
-        Then colors -> statusbar.bg should be yellow
 
     Scenario: Deleting an open tab via the completion
         Given I have a fresh instance

tests/end2end/features/history.feature

@@ -11,44 +11,44 @@ Feature: Page history
     Scenario: Simple history saving
         When I open data/numbers/1.txt
         And I open data/numbers/2.txt
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/numbers/1.txt
             http://localhost:(port)/data/numbers/2.txt
 
     Scenario: History item with title
         When I open data/title.html
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/title.html Test title
 
     Scenario: History item with redirect
         When I open redirect-to?url=data/title.html without waiting
         And I wait until data/title.html is loaded
-        Then the history file should contain:
+        Then the history should contain:
             r http://localhost:(port)/redirect-to?url=data/title.html Test title
             http://localhost:(port)/data/title.html Test title
 
     Scenario: History item with spaces in URL
         When I open data/title with spaces.html
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/title%20with%20spaces.html Test title
 
     Scenario: History item with umlauts
         When I open data/äöü.html
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/%C3%A4%C3%B6%C3%BC.html Chäschüechli
 
     @flaky @qtwebengine_todo: Error page message is not implemented
     Scenario: History with an error
         When I run :open file:///does/not/exist
         And I wait for "Error while loading file:///does/not/exist: Error opening /does/not/exist: *" in the log
-        Then the history file should contain:
+        Then the history should contain:
             file:///does/not/exist Error loading page: file:///does/not/exist
 
     @qtwebengine_todo: Error page message is not implemented
     Scenario: History with a 404
         When I open status/404 without waiting
         And I wait for "Error while loading http://localhost:*/status/404: NOT FOUND" in the log
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/status/404 Error loading page: http://localhost:(port)/status/404
 
     Scenario: History with invalid URL
@@ -61,32 +61,32 @@ Feature: Page history
         When I open data/data_link.html
         And I run :click-element id link
         And I wait until data:;base64,cXV0ZWJyb3dzZXI= is loaded
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/data_link.html data: link
 
     Scenario: History with view-source URL
         When I open data/title.html
         And I run :view-source
         And I wait for "Changing title for idx * to 'Source for http://localhost:*/data/title.html'" in the log
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/title.html Test title
 
     Scenario: Clearing history
         When I open data/title.html
         And I run :history-clear --force
-        Then the history file should be empty
+        Then the history should be empty
 
     Scenario: Clearing history with confirmation
         When I open data/title.html
         And I run :history-clear
         And I wait for "Asking question <* title='Clear all browsing history?'>, *" in the log
         And I run :prompt-accept yes
-        Then the history file should be empty
+        Then the history should be empty
 
     Scenario: History with yanked URL and 'add to history' flag
         When I open data/hints/html/simple.html
         And I hint with args "--add-history links yank" and follow a
-        Then the history file should contain:
+        Then the history should contain:
             http://localhost:(port)/data/hints/html/simple.html Simple link
             http://localhost:(port)/data/hello.txt
 

tests/end2end/features/misc.feature

@@ -702,3 +702,9 @@ Feature: Various utility commands.
         And I wait for "Renderer process was killed" in the log
         And I open data/numbers/3.txt
         Then no crash should happen
+
+    ## Other
+
+    Scenario: Open qute://version
+        When I open qute://version
+        Then the page should contain the plaintext "Version info"

tests/end2end/features/test_completion_bdd.py

@@ -24,5 +24,5 @@ bdd.scenarios('completion.feature')
 @bdd.then(bdd.parsers.parse("the completion model should be {model}"))
 def check_model(quteproc, model):
     """Make sure the completion model was set to something."""
-    pattern = "Setting completion model to {} with pattern *".format(model)
+    pattern = "Starting {} completion *".format(model)
     quteproc.wait_for(message=pattern)

tests/end2end/features/test_history_bdd.py

@@ -17,36 +17,29 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-import os.path
+import logging
+import re
 
 import pytest_bdd as bdd
+
 bdd.scenarios('history.feature')
 
 
-@bdd.then(bdd.parsers.parse("the history file should contain:\n{expected}"))
+@bdd.then(bdd.parsers.parse("the history should contain:\n{expected}"))
-def check_history(quteproc, httpbin, expected):
+def check_history(quteproc, httpbin, tmpdir, expected):
-    history_file = os.path.join(quteproc.basedir, 'data', 'history')
+    path = tmpdir / 'history'
-    quteproc.send_cmd(':save history')
+    quteproc.send_cmd(':debug-dump-history "{}"'.format(path))
-    quteproc.wait_for(message=':save saved history')
+    quteproc.wait_for(category='message', loglevel=logging.INFO,
+                      message='Dumped history to {}'.format(path))
 
-    expected = expected.replace('(port)', str(httpbin.port)).splitlines()
+    with path.open('r', encoding='utf-8') as f:
+        # ignore access times, they will differ in each run
+        actual = '\n'.join(re.sub('^\\d+-?', '', line).strip() for line in f)
 
-    with open(history_file, 'r', encoding='utf-8') as f:
+    expected = expected.replace('(port)', str(httpbin.port))
-        lines = []
+    assert actual == expected
-        for line in f:
-            if not line.strip():
-                continue
-            print('history line: ' + line)
-            atime, line = line.split(' ', maxsplit=1)
-            line = line.rstrip()
-            if '-' in atime:
-                flags = atime.split('-')[1]
-                line = '{} {}'.format(flags, line)
-            lines.append(line)
-
-    assert lines == expected
 
 
-@bdd.then("the history file should be empty")
+@bdd.then("the history should be empty")
-def check_history_empty(quteproc, httpbin):
+def check_history_empty(quteproc, httpbin, tmpdir):
-    check_history(quteproc, httpbin, '')
+    check_history(quteproc, httpbin, tmpdir, '')

tests/helpers/fixtures.py

@@ -41,7 +41,7 @@ import helpers.stubs as stubsmod
 from qutebrowser.config import config
 from qutebrowser.utils import objreg, standarddir
 from qutebrowser.browser.webkit import cookies
-from qutebrowser.misc import savemanager
+from qutebrowser.misc import savemanager, sql
 from qutebrowser.keyinput import modeman
 
 from PyQt5.QtCore import PYQT_VERSION, pyqtSignal, QEvent, QSize, Qt, QObject
@@ -257,18 +257,9 @@ def bookmark_manager_stub(stubs):
     objreg.delete('bookmark-manager')
 
 
-@pytest.fixture
-def web_history_stub(stubs):
-    """Fixture which provides a fake web-history object."""
-    stub = stubs.WebHistoryStub()
-    objreg.register('web-history', stub)
-    yield stub
-    objreg.delete('web-history')
-
-
 @pytest.fixture
 def session_manager_stub(stubs):
-    """Fixture which provides a fake web-history object."""
+    """Fixture which provides a fake session-manager object."""
     stub = stubs.SessionManagerStub()
     objreg.register('session-manager', stub)
     yield stub
@@ -482,3 +473,37 @@ def short_tmpdir():
     """A short temporary directory for a XDG_RUNTIME_DIR."""
     with tempfile.TemporaryDirectory() as tdir:
         yield py.path.local(tdir)  # pylint: disable=no-member
+
+
+@pytest.fixture
+def init_sql(data_tmpdir):
+    """Initialize the SQL module, and shut it down after the test."""
+    path = str(data_tmpdir / 'test.db')
+    sql.init(path)
+    yield
+    sql.close()
+
+
+class ModelValidator:
+
+    """Validates completion models."""
+
+    def __init__(self, modeltester):
+        modeltester.data_display_may_return_none = True
+        self._model = None
+        self._modeltester = modeltester
+
+    def set_model(self, model):
+        self._model = model
+        self._modeltester.check(model)
+
+    def validate(self, expected):
+        assert self._model.rowCount() == len(expected)
+        for row, items in enumerate(expected):
+            for col, item in enumerate(items):
+                assert self._model.data(self._model.index(row, col)) == item
+
+
+@pytest.fixture
+def model_validator(qtmodeltester):
+    return ModelValidator(qtmodeltester)

tests/helpers/stubs.py

@@ -29,7 +29,7 @@ from PyQt5.QtNetwork import (QNetworkRequest, QAbstractNetworkCache,
                              QNetworkCacheMetaData)
 from PyQt5.QtWidgets import QCommonStyle, QLineEdit, QWidget, QTabBar
 
-from qutebrowser.browser import browsertab, history
+from qutebrowser.browser import browsertab
 from qutebrowser.config import configexc
 from qutebrowser.utils import usertypes, utils
 from qutebrowser.mainwindow import mainwindow
@@ -405,6 +405,10 @@ class InstaTimer(QObject):
     def setInterval(self, interval):
         pass
 
+    @staticmethod
+    def singleShot(_interval, fun):
+        fun()
+
 
 class FakeConfigType:
 
@@ -538,24 +542,6 @@ class QuickmarkManagerStub(UrlMarkManagerStub):
         self.delete(key)
 
 
-class WebHistoryStub(QObject):
-
-    """Stub for the web-history object."""
-
-    add_completion_item = pyqtSignal(history.Entry)
-    cleared = pyqtSignal()
-
-    def __init__(self, parent=None):
-        super().__init__(parent)
-        self.history_dict = collections.OrderedDict()
-
-    def __iter__(self):
-        return iter(self.history_dict.values())
-
-    def __len__(self):
-        return len(self.history_dict)
-
-
 class HostBlockerStub:
 
     """Stub for the host-blocker object."""

tests/unit/browser/test_history.py

@@ -0,0 +1,349 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Tests for the global page history."""
+
+import logging
+
+import pytest
+from PyQt5.QtCore import QUrl
+
+from qutebrowser.browser import history
+from qutebrowser.utils import objreg, urlutils, usertypes
+from qutebrowser.commands import cmdexc
+
+
+@pytest.fixture(autouse=True)
+def prerequisites(config_stub, fake_save_manager, init_sql):
+    """Make sure everything is ready to initialize a WebHistory."""
+    config_stub.data = {'general': {'private-browsing': False}}
+
+
+@pytest.fixture()
+def hist(tmpdir):
+    return history.WebHistory()
+
+
+@pytest.fixture()
+def mock_time(mocker):
+    m = mocker.patch('qutebrowser.browser.history.time')
+    m.time.return_value = 12345
+    return 12345
+
+
+def test_iter(hist):
+    urlstr = 'http://www.example.com/'
+    url = QUrl(urlstr)
+    hist.add_url(url, atime=12345)
+
+    assert list(hist) == [(urlstr, '', 12345, False)]
+
+
+def test_len(hist):
+    assert len(hist) == 0
+
+    url = QUrl('http://www.example.com/')
+    hist.add_url(url)
+
+    assert len(hist) == 1
+
+
+def test_contains(hist):
+    hist.add_url(QUrl('http://www.example.com/'), title='Title', atime=12345)
+    assert 'http://www.example.com/' in hist
+    assert 'www.example.com' not in hist
+    assert 'Title' not in hist
+    assert 12345 not in hist
+
+
+def test_get_recent(hist):
+    hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890)
+    hist.add_url(QUrl('http://example.com/'), atime=12345)
+    assert list(hist.get_recent()) == [
+        ('http://www.qutebrowser.org/', '', 67890, False),
+        ('http://example.com/', '', 12345, False),
+    ]
+
+
+def test_entries_between(hist):
+    hist.add_url(QUrl('http://www.example.com/1'), atime=12345)
+    hist.add_url(QUrl('http://www.example.com/2'), atime=12346)
+    hist.add_url(QUrl('http://www.example.com/3'), atime=12347)
+    hist.add_url(QUrl('http://www.example.com/4'), atime=12348)
+    hist.add_url(QUrl('http://www.example.com/5'), atime=12348)
+    hist.add_url(QUrl('http://www.example.com/6'), atime=12349)
+    hist.add_url(QUrl('http://www.example.com/7'), atime=12350)
+
+    times = [x.atime for x in hist.entries_between(12346, 12349)]
+    assert times == [12349, 12348, 12348, 12347]
+
+
+def test_entries_before(hist):
+    hist.add_url(QUrl('http://www.example.com/1'), atime=12346)
+    hist.add_url(QUrl('http://www.example.com/2'), atime=12346)
+    hist.add_url(QUrl('http://www.example.com/3'), atime=12347)
+    hist.add_url(QUrl('http://www.example.com/4'), atime=12348)
+    hist.add_url(QUrl('http://www.example.com/5'), atime=12348)
+    hist.add_url(QUrl('http://www.example.com/6'), atime=12348)
+    hist.add_url(QUrl('http://www.example.com/7'), atime=12349)
+    hist.add_url(QUrl('http://www.example.com/8'), atime=12349)
+
+    times = [x.atime for x in hist.entries_before(12348, limit=3, offset=2)]
+    assert times == [12348, 12347, 12346]
+
+
+def test_clear(qtbot, tmpdir, hist, mocker):
+    hist.add_url(QUrl('http://example.com/'))
+    hist.add_url(QUrl('http://www.qutebrowser.org/'))
+
+    m = mocker.patch('qutebrowser.browser.history.message.confirm_async',
+                     new=mocker.Mock, spec=[])
+    hist.clear()
+    assert m.called
+
+
+def test_clear_force(qtbot, tmpdir, hist):
+    hist.add_url(QUrl('http://example.com/'))
+    hist.add_url(QUrl('http://www.qutebrowser.org/'))
+    hist.clear(force=True)
+    assert not len(hist)
+    assert not len(hist.completion)
+
+
+def test_delete_url(hist):
+    hist.add_url(QUrl('http://example.com/'), atime=0)
+    hist.add_url(QUrl('http://example.com/1'), atime=0)
+    hist.add_url(QUrl('http://example.com/2'), atime=0)
+
+    before = set(hist)
+    completion_before = set(hist.completion)
+
+    hist.delete_url(QUrl('http://example.com/1'))
+
+    diff = before.difference(set(hist))
+    assert diff == {('http://example.com/1', '', 0, False)}
+
+    completion_diff = completion_before.difference(set(hist.completion))
+    assert completion_diff == {('http://example.com/1', '', 0)}
+
+
+@pytest.mark.parametrize('url, atime, title, redirect, expected_url', [
+    ('http://www.example.com', 12346, 'the title', False,
+        'http://www.example.com'),
+    ('http://www.example.com', 12346, 'the title', True,
+        'http://www.example.com'),
+    ('http://www.example.com/spa ce', 12346, 'the title', False,
+        'http://www.example.com/spa%20ce'),
+    ('https://user:pass@example.com', 12346, 'the title', False,
+        'https://user@example.com'),
+])
+def test_add_url(qtbot, hist, url, atime, title, redirect, expected_url):
+    hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect)
+    assert list(hist) == [(expected_url, title, atime, redirect)]
+    if redirect:
+        assert not len(hist.completion)
+    else:
+        assert list(hist.completion) == [(expected_url, title, atime)]
+
+
+def test_add_url_invalid(qtbot, hist, caplog):
+    with caplog.at_level(logging.WARNING):
+        hist.add_url(QUrl())
+    assert not list(hist)
+    assert not list(hist.completion)
+
+
+@pytest.mark.parametrize('level, url, req_url, expected', [
+    (logging.DEBUG, 'a.com', 'a.com', [('a.com', 'title', 12345, False)]),
+    (logging.DEBUG, 'a.com', 'b.com', [('a.com', 'title', 12345, False),
+                                       ('b.com', 'title', 12345, True)]),
+    (logging.WARNING, 'a.com', '', [('a.com', 'title', 12345, False)]),
+    (logging.WARNING, '', '', []),
+    (logging.WARNING, 'data:foo', '', []),
+    (logging.WARNING, 'a.com', 'data:foo', []),
+])
+def test_add_from_tab(hist, level, url, req_url, expected, mock_time, caplog):
+    with caplog.at_level(level):
+        hist.add_from_tab(QUrl(url), QUrl(req_url), 'title')
+    assert set(hist) == set(expected)
+
+
+@pytest.fixture
+def hist_interface(hist):
+    # pylint: disable=invalid-name
+    QtWebKit = pytest.importorskip('PyQt5.QtWebKit')
+    from qutebrowser.browser.webkit import webkithistory
+    QWebHistoryInterface = QtWebKit.QWebHistoryInterface
+    # pylint: enable=invalid-name
+    hist.add_url(url=QUrl('http://www.example.com/'), title='example')
+    interface = webkithistory.WebHistoryInterface(hist)
+    QWebHistoryInterface.setDefaultInterface(interface)
+    yield
+    QWebHistoryInterface.setDefaultInterface(None)
+
+
+def test_history_interface(qtbot, webview, hist_interface):
+    html = b"<a href='about:blank'>foo</a>"
+    url = urlutils.data_url('text/html', html)
+    with qtbot.waitSignal(webview.loadFinished):
+        webview.load(url)
+
+
+@pytest.fixture
+def cleanup_init():
+    # prevent test_init from leaking state
+    yield
+    hist = objreg.get('web-history', None)
+    if hist is not None:
+        hist.setParent(None)
+        objreg.delete('web-history')
+    try:
+        from PyQt5.QtWebKit import QWebHistoryInterface
+        QWebHistoryInterface.setDefaultInterface(None)
+    except ImportError:
+        pass
+
+
+@pytest.mark.parametrize('backend', [usertypes.Backend.QtWebEngine,
+                                     usertypes.Backend.QtWebKit])
+def test_init(backend, qapp, tmpdir, monkeypatch, cleanup_init):
+    if backend == usertypes.Backend.QtWebKit:
+        pytest.importorskip('PyQt5.QtWebKitWidgets')
+    else:
+        assert backend == usertypes.Backend.QtWebEngine
+
+    monkeypatch.setattr(history.objects, 'backend', backend)
+    history.init(qapp)
+    hist = objreg.get('web-history')
+    assert hist.parent() is qapp
+
+    try:
+        from PyQt5.QtWebKit import QWebHistoryInterface
+    except ImportError:
+        QWebHistoryInterface = None
+
+    if backend == usertypes.Backend.QtWebKit:
+        default_interface = QWebHistoryInterface.defaultInterface()
+        assert default_interface._history is hist
+    else:
+        assert backend == usertypes.Backend.QtWebEngine
+        if QWebHistoryInterface is None:
+            default_interface = None
+        else:
+            default_interface = QWebHistoryInterface.defaultInterface()
+        # For this to work, nothing can ever have called setDefaultInterface
+        # before (so we need to test webengine before webkit)
+        assert default_interface is None
+
+
+def test_import_txt(hist, data_tmpdir, monkeypatch, stubs):
+    monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer)
+    histfile = data_tmpdir / 'history'
+    # empty line is deliberate, to test skipping empty lines
+    histfile.write('''12345 http://example.com/ title
+                      12346 http://qutebrowser.org/
+                      67890 http://example.com/path
+
+                      68891-r http://example.com/path/other ''')
+
+    hist.import_txt()
+
+    assert list(hist) == [
+        ('http://example.com/', 'title', 12345, False),
+        ('http://qutebrowser.org/', '', 12346, False),
+        ('http://example.com/path', '', 67890, False),
+        ('http://example.com/path/other', '', 68891, True)
+    ]
+
+    assert not histfile.exists()
+    assert (data_tmpdir / 'history.bak').exists()
+
+
+@pytest.mark.parametrize('line', [
+    '',
+    '#12345 http://example.com/commented',
+
+    # https://bugreports.qt.io/browse/QTBUG-60364
+    '12345 http://.com/',
+    '12345 https://www..com/',
+
+    # issue #2646
+    '12345 data:text/html;charset=UTF-8,%3C%21DOCTYPE%20html%20PUBLIC%20%22-',
+])
+def test_import_txt_skip(hist, data_tmpdir, line, monkeypatch, stubs):
+    """import_txt should skip certain lines silently."""
+    monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer)
+    histfile = data_tmpdir / 'history'
+    histfile.write(line)
+
+    hist.import_txt()
+
+    assert not histfile.exists()
+    assert not len(hist)
+
+
+@pytest.mark.parametrize('line', [
+    'xyz http://example.com/bad-timestamp',
+    '12345',
+    'http://example.com/no-timestamp',
+    '68891-r-r http://example.com/double-flag',
+    '68891-x http://example.com/bad-flag',
+    '68891 http://.com',
+])
+def test_import_txt_invalid(hist, data_tmpdir, line, monkeypatch, stubs,
+                            caplog):
+    """import_txt should fail on certain lines."""
+    monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer)
+    histfile = data_tmpdir / 'history'
+    histfile.write(line)
+
+    with caplog.at_level(logging.ERROR):
+        hist.import_txt()
+
+    assert any(rec.msg.startswith("Failed to import history:")
+               for rec in caplog.records)
+
+    assert histfile.exists()
+
+
+def test_import_txt_nonexistent(hist, data_tmpdir, monkeypatch, stubs):
+    """import_txt should do nothing if the history file doesn't exist."""
+    monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer)
+    hist.import_txt()
+
+
+def test_debug_dump_history(hist, tmpdir):
+    hist.add_url(QUrl('http://example.com/1'), title="Title1", atime=12345)
+    hist.add_url(QUrl('http://example.com/2'), title="Title2", atime=12346)
+    hist.add_url(QUrl('http://example.com/3'), title="Title3", atime=12347)
+    hist.add_url(QUrl('http://example.com/4'), title="Title4", atime=12348,
+                 redirect=True)
+    histfile = tmpdir / 'history'
+    hist.debug_dump_history(str(histfile))
+    expected = ['12345 http://example.com/1 Title1',
+                '12346 http://example.com/2 Title2',
+                '12347 http://example.com/3 Title3',
+                '12348-r http://example.com/4 Title4']
+    assert histfile.read() == '\n'.join(expected)
+
+
+def test_debug_dump_history_nonexistent(hist, tmpdir):
+    histfile = tmpdir / 'nonexistent' / 'history'
+    with pytest.raises(cmdexc.CommandError):
+        hist.debug_dump_history(str(histfile))

tests/unit/browser/test_qutescheme.py

@@ -89,16 +89,17 @@ class TestHistoryHandler:
         items = []
         for i in range(entry_count):
             entry_atime = now - i * interval
-            entry = history.Entry(atime=str(entry_atime),
+            entry = {"atime": str(entry_atime),
-                url=QUrl("www.x.com/" + str(i)), title="Page " + str(i))
+                     "url": QUrl("www.x.com/" + str(i)),
+                     "title": "Page " + str(i)}
             items.insert(0, entry)
 
         return items
 
     @pytest.fixture
-    def fake_web_history(self, fake_save_manager, tmpdir):
+    def fake_web_history(self, fake_save_manager, tmpdir, init_sql):
         """Create a fake web-history and register it into objreg."""
-        web_history = history.WebHistory(tmpdir.dirname, 'fake-history')
+        web_history = history.WebHistory()
         objreg.register('web-history', web_history)
         yield web_history
         objreg.delete('web-history')
@@ -107,8 +108,7 @@ class TestHistoryHandler:
     def fake_history(self, fake_web_history, entries):
         """Create fake history."""
         for item in entries:
-            fake_web_history._add_entry(item)
+            fake_web_history.add_url(**item)
-        fake_web_history.save()
 
     @pytest.mark.parametrize("start_time_offset, expected_item_count", [
         (0, 4),
@@ -123,45 +123,25 @@ class TestHistoryHandler:
         url = QUrl("qute://history/data?start_time=" + str(start_time))
         _mimetype, data = qutescheme.qute_history(url)
         items = json.loads(data)
-        items = [item for item in items if 'time' in item]  # skip 'next' item
 
         assert len(items) == expected_item_count
 
         # test times
         end_time = start_time - 24*60*60
         for item in items:
-            assert item['time'] <= start_time * 1000
+            assert item['time'] <= start_time
-            assert item['time'] > end_time * 1000
+            assert item['time'] > end_time
-
-    @pytest.mark.parametrize("start_time_offset, next_time", [
-        (0, 24*60*60),
-        (24*60*60, 48*60*60),
-        (48*60*60, -1),
-        (72*60*60, -1)
-    ])
-    def test_qutehistory_next(self, start_time_offset, next_time, now):
-        """Ensure qute://history/data returns correct items."""
-        start_time = now - start_time_offset
-        url = QUrl("qute://history/data?start_time=" + str(start_time))
-        _mimetype, data = qutescheme.qute_history(url)
-        items = json.loads(data)
-        items = [item for item in items if 'next' in item]  # 'next' items
-        assert len(items) == 1
-
-        if next_time == -1:
-            assert items[0]["next"] == -1
-        else:
-            assert items[0]["next"] == now - next_time
 
     def test_qute_history_benchmark(self, fake_web_history, benchmark, now):
-        # items must be earliest-first to ensure history is sorted properly
+        r = range(100000)
-        for t in range(100000, 0, -1):  # one history per second
+        entries = {
-            entry = history.Entry(
+            'atime': [int(now - t) for t in r],
-                atime=str(now - t),
+            'url': ['www.x.com/{}'.format(t) for t in r],
-                url=QUrl('www.x.com/{}'.format(t)),
+            'title': ['x at {}'.format(t) for t in r],
-                title='x at {}'.format(t))
+            'redirect': [False for _ in r],
-            fake_web_history._add_entry(entry)
+        }
 
+        fake_web_history.insert_batch(entries)
         url = QUrl("qute://history/data?start_time={}".format(now))
         _mimetype, data = benchmark(qutescheme.qute_history, url)
         assert len(json.loads(data)) > 1

tests/unit/browser/webkit/test_history.py

@@ -1,383 +0,0 @@
-# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
-
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Tests for the global page history."""
-
-import logging
-
-import pytest
-import hypothesis
-from hypothesis import strategies
-from PyQt5.QtCore import QUrl
-
-from qutebrowser.browser import history
-from qutebrowser.utils import objreg, urlutils, usertypes
-
-
-class FakeWebHistory:
-
-    """A fake WebHistory object."""
-
-    def __init__(self, history_dict):
-        self.history_dict = history_dict
-
-
-@pytest.fixture()
-def hist(tmpdir, fake_save_manager):
-    return history.WebHistory(hist_dir=str(tmpdir), hist_name='history')
-
-
-def test_async_read_twice(monkeypatch, qtbot, tmpdir, caplog,
-                          fake_save_manager):
-    (tmpdir / 'filled-history').write('\n'.join([
-        '12345 http://example.com/ title',
-        '67890 http://example.com/',
-        '12345 http://qutebrowser.org/ blah',
-    ]))
-    hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history')
-    next(hist.async_read())
-    with pytest.raises(StopIteration):
-        next(hist.async_read())
-    expected = "Ignoring async_read() because reading is started."
-    assert len(caplog.records) == 1
-    assert caplog.records[0].msg == expected
-
-
-@pytest.mark.parametrize('redirect', [True, False])
-def test_adding_item_during_async_read(qtbot, hist, redirect):
-    """Check what happens when adding URL while reading the history."""
-    url = QUrl('http://www.example.com/')
-
-    with qtbot.assertNotEmitted(hist.add_completion_item), \
-            qtbot.assertNotEmitted(hist.item_added):
-        hist.add_url(url, redirect=redirect, atime=12345)
-
-    if redirect:
-        with qtbot.assertNotEmitted(hist.add_completion_item):
-            with qtbot.waitSignal(hist.async_read_done):
-                list(hist.async_read())
-    else:
-        with qtbot.waitSignals([hist.add_completion_item,
-                                hist.async_read_done], order='strict'):
-            list(hist.async_read())
-
-    assert not hist._temp_history
-
-    expected = history.Entry(url=url, atime=12345, redirect=redirect, title="")
-    assert list(hist.history_dict.values()) == [expected]
-
-
-def test_iter(hist):
-    list(hist.async_read())
-
-    url = QUrl('http://www.example.com/')
-    hist.add_url(url, atime=12345)
-
-    entry = history.Entry(url=url, atime=12345, redirect=False, title="")
-    assert list(hist) == [entry]
-
-
-def test_len(hist):
-    assert len(hist) == 0
-    list(hist.async_read())
-
-    url = QUrl('http://www.example.com/')
-    hist.add_url(url)
-
-    assert len(hist) == 1
-
-
-@pytest.mark.parametrize('line', [
-    '12345 http://example.com/ title',  # with title
-    '67890 http://example.com/',  # no title
-    '12345 http://qutebrowser.org/ ',  # trailing space
-    ' ',
-    '',
-])
-def test_read(hist, tmpdir, line):
-    (tmpdir / 'filled-history').write(line + '\n')
-    hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history')
-    list(hist.async_read())
-
-
-def test_updated_entries(hist, tmpdir):
-    (tmpdir / 'filled-history').write('12345 http://example.com/\n'
-                                      '67890 http://example.com/\n')
-    hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history')
-    list(hist.async_read())
-
-    assert hist.history_dict['http://example.com/'].atime == 67890
-    hist.add_url(QUrl('http://example.com/'), atime=99999)
-    assert hist.history_dict['http://example.com/'].atime == 99999
-
-
-def test_invalid_read(hist, tmpdir, caplog):
-    (tmpdir / 'filled-history').write('foobar\n12345 http://example.com/')
-    hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history')
-    with caplog.at_level(logging.WARNING):
-        list(hist.async_read())
-
-    entries = list(hist.history_dict.values())
-
-    assert len(entries) == 1
-    assert len(caplog.records) == 1
-    msg = "Invalid history entry 'foobar': 2 or 3 fields expected!"
-    assert caplog.records[0].msg == msg
-
-
-def test_get_recent(hist, tmpdir):
-    (tmpdir / 'filled-history').write('12345 http://example.com/')
-    hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history')
-    list(hist.async_read())
-
-    hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890)
-    lines = hist.get_recent()
-
-    expected = ['12345 http://example.com/',
-                '67890 http://www.qutebrowser.org/']
-    assert lines == expected
-
-
-def test_save(hist, tmpdir):
-    hist_file = tmpdir / 'filled-history'
-    hist_file.write('12345 http://example.com/\n')
-
-    hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history')
-    list(hist.async_read())
-
-    hist.add_url(QUrl('http://www.qutebrowser.org/'), atime=67890)
-    hist.save()
-
-    lines = hist_file.read().splitlines()
-    expected = ['12345 http://example.com/',
-                '67890 http://www.qutebrowser.org/']
-    assert lines == expected
-
-    hist.add_url(QUrl('http://www.the-compiler.org/'), atime=99999)
-    hist.save()
-    expected.append('99999 http://www.the-compiler.org/')
-
-    lines = hist_file.read().splitlines()
-    assert lines == expected
-
-
-def test_clear(qtbot, hist, tmpdir):
-    hist_file = tmpdir / 'filled-history'
-    hist_file.write('12345 http://example.com/\n')
-
-    hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history')
-    list(hist.async_read())
-
-    hist.add_url(QUrl('http://www.qutebrowser.org/'))
-
-    with qtbot.waitSignal(hist.cleared):
-        hist._do_clear()
-
-    assert not hist_file.read()
-    assert not hist.history_dict
-    assert not hist._new_history
-
-    hist.add_url(QUrl('http://www.the-compiler.org/'), atime=67890)
-    hist.save()
-
-    lines = hist_file.read().splitlines()
-    assert lines == ['67890 http://www.the-compiler.org/']
-
-
-def test_add_item(qtbot, hist):
-    list(hist.async_read())
-    url = 'http://www.example.com/'
-
-    with qtbot.waitSignals([hist.add_completion_item, hist.item_added],
-                           order='strict'):
-        hist.add_url(QUrl(url), atime=12345, title="the title")
-
-    entry = history.Entry(url=QUrl(url), redirect=False, atime=12345,
-                          title="the title")
-    assert hist.history_dict[url] == entry
-
-
-def test_add_item_redirect(qtbot, hist):
-    list(hist.async_read())
-    url = 'http://www.example.com/'
-    with qtbot.assertNotEmitted(hist.add_completion_item):
-        with qtbot.waitSignal(hist.item_added):
-            hist.add_url(QUrl(url), redirect=True, atime=12345)
-
-    entry = history.Entry(url=QUrl(url), redirect=True, atime=12345, title="")
-    assert hist.history_dict[url] == entry
-
-
-def test_add_item_redirect_update(qtbot, tmpdir, fake_save_manager):
-    """A redirect update added should override a non-redirect one."""
-    url = 'http://www.example.com/'
-
-    hist_file = tmpdir / 'filled-history'
-    hist_file.write('12345 {}\n'.format(url))
-    hist = history.WebHistory(hist_dir=str(tmpdir), hist_name='filled-history')
-    list(hist.async_read())
-
-    with qtbot.assertNotEmitted(hist.add_completion_item):
-        with qtbot.waitSignal(hist.item_added):
-            hist.add_url(QUrl(url), redirect=True, atime=67890)
-
-    entry = history.Entry(url=QUrl(url), redirect=True, atime=67890, title="")
-    assert hist.history_dict[url] == entry
-
-
-@pytest.mark.parametrize('line, expected', [
-    (
-        # old format without title
-        '12345 http://example.com/',
-        history.Entry(atime=12345, url=QUrl('http://example.com/'), title='',)
-    ),
-    (
-        # trailing space without title
-        '12345 http://example.com/ ',
-        history.Entry(atime=12345, url=QUrl('http://example.com/'), title='',)
-    ),
-    (
-        # new format with title
-        '12345 http://example.com/ this is a title',
-        history.Entry(atime=12345, url=QUrl('http://example.com/'),
-                      title='this is a title')
-    ),
-    (
-        # weird NUL bytes
-        '\x0012345 http://example.com/',
-        history.Entry(atime=12345, url=QUrl('http://example.com/'), title=''),
-    ),
-    (
-        # redirect flag
-        '12345-r http://example.com/ this is a title',
-        history.Entry(atime=12345, url=QUrl('http://example.com/'),
-                      title='this is a title', redirect=True)
-    ),
-])
-def test_entry_parse_valid(line, expected):
-    entry = history.Entry.from_str(line)
-    assert entry == expected
-
-
-@pytest.mark.parametrize('line', [
-    '12345',  # one field
-    '12345 ::',  # invalid URL
-    'xyz http://www.example.com/',  # invalid timestamp
-    '12345-x http://www.example.com/',  # invalid flags
-    '12345-r-r http://www.example.com/',  # double flags
-])
-def test_entry_parse_invalid(line):
-    with pytest.raises(ValueError):
-        history.Entry.from_str(line)
-
-
-@hypothesis.given(strategies.text())
-def test_entry_parse_hypothesis(text):
-    """Make sure parsing works or gives us ValueError."""
-    try:
-        history.Entry.from_str(text)
-    except ValueError:
-        pass
-
-
-@pytest.mark.parametrize('entry, expected', [
-    # simple
-    (
-        history.Entry(12345, QUrl('http://example.com/'), "the title"),
-        "12345 http://example.com/ the title",
-    ),
-    # timestamp as float
-    (
-        history.Entry(12345.678, QUrl('http://example.com/'), "the title"),
-        "12345 http://example.com/ the title",
-    ),
-    # no title
-    (
-        history.Entry(12345.678, QUrl('http://example.com/'), ""),
-        "12345 http://example.com/",
-    ),
-    # redirect flag
-    (
-        history.Entry(12345.678, QUrl('http://example.com/'), "",
-                      redirect=True),
-        "12345-r http://example.com/",
-    ),
-])
-def test_entry_str(entry, expected):
-    assert str(entry) == expected
-
-
-@pytest.fixture
-def hist_interface():
-    # pylint: disable=invalid-name
-    QtWebKit = pytest.importorskip('PyQt5.QtWebKit')
-    from qutebrowser.browser.webkit import webkithistory
-    QWebHistoryInterface = QtWebKit.QWebHistoryInterface
-    # pylint: enable=invalid-name
-    entry = history.Entry(atime=0, url=QUrl('http://www.example.com/'),
-                          title='example')
-    history_dict = {'http://www.example.com/': entry}
-    fake_hist = FakeWebHistory(history_dict)
-    interface = webkithistory.WebHistoryInterface(fake_hist)
-    QWebHistoryInterface.setDefaultInterface(interface)
-    yield
-    QWebHistoryInterface.setDefaultInterface(None)
-
-
-def test_history_interface(qtbot, webview, hist_interface):
-    html = b"<a href='about:blank'>foo</a>"
-    url = urlutils.data_url('text/html', html)
-    with qtbot.waitSignal(webview.loadFinished):
-        webview.load(url)
-
-
-@pytest.mark.parametrize('backend', [usertypes.Backend.QtWebEngine,
-                                     usertypes.Backend.QtWebKit])
-def test_init(backend, qapp, tmpdir, monkeypatch, fake_save_manager):
-    if backend == usertypes.Backend.QtWebKit:
-        pytest.importorskip('PyQt5.QtWebKitWidgets')
-    else:
-        assert backend == usertypes.Backend.QtWebEngine
-
-    monkeypatch.setattr(history.standarddir, 'data', lambda: str(tmpdir))
-    monkeypatch.setattr(history.objects, 'backend', backend)
-    history.init(qapp)
-    hist = objreg.get('web-history')
-    assert hist.parent() is qapp
-
-    try:
-        from PyQt5.QtWebKit import QWebHistoryInterface
-    except ImportError:
-        QWebHistoryInterface = None
-
-    if backend == usertypes.Backend.QtWebKit:
-        default_interface = QWebHistoryInterface.defaultInterface()
-        assert default_interface._history is hist
-    else:
-        assert backend == usertypes.Backend.QtWebEngine
-        if QWebHistoryInterface is None:
-            default_interface = None
-        else:
-            default_interface = QWebHistoryInterface.defaultInterface()
-        # For this to work, nothing can ever have called setDefaultInterface
-        # before (so we need to test webengine before webkit)
-        assert default_interface is None
-
-    assert fake_save_manager.add_saveable.called
-    objreg.delete('web-history')

tests/unit/completion/test_column_widths.py

@@ -1,50 +0,0 @@
-# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
-
-# Copyright 2015-2017 Alexander Cogneau <alexander.cogneau@gmail.com>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Tests for qutebrowser.completion.models column widths."""
-
-import pytest
-
-from qutebrowser.completion.models.base import BaseCompletionModel
-from qutebrowser.completion.models.configmodel import (
-    SettingOptionCompletionModel, SettingSectionCompletionModel,
-    SettingValueCompletionModel)
-from qutebrowser.completion.models.miscmodels import (
-    CommandCompletionModel, HelpCompletionModel, QuickmarkCompletionModel,
-    BookmarkCompletionModel, SessionCompletionModel)
-from qutebrowser.completion.models.urlmodel import UrlCompletionModel
-
-
-CLASSES = [BaseCompletionModel, SettingOptionCompletionModel,
-           SettingOptionCompletionModel, SettingSectionCompletionModel,
-           SettingValueCompletionModel, CommandCompletionModel,
-           HelpCompletionModel, QuickmarkCompletionModel,
-           BookmarkCompletionModel, SessionCompletionModel, UrlCompletionModel]
-
-
-@pytest.mark.parametrize("model", CLASSES)
-def test_list_size(model):
-    """Test if there are 3 items in the COLUMN_WIDTHS property."""
-    assert len(model.COLUMN_WIDTHS) == 3
-
-
-@pytest.mark.parametrize("model", CLASSES)
-def test_column_width_sum(model):
-    """Test if the sum of the widths asserts to 100."""
-    assert sum(model.COLUMN_WIDTHS) == 100

tests/unit/completion/test_completer.py

@@ -26,7 +26,6 @@ from PyQt5.QtCore import QObject
 from PyQt5.QtGui import QStandardItemModel
 
 from qutebrowser.completion import completer
-from qutebrowser.utils import usertypes
 from qutebrowser.commands import command, cmdutils
 
 
@@ -34,11 +33,10 @@ class FakeCompletionModel(QStandardItemModel):
 
     """Stub for a completion model."""
 
-    DUMB_SORT = None
+    def __init__(self, kind, *pos_args, parent=None):
-
-    def __init__(self, kind, parent=None):
         super().__init__(parent)
         self.kind = kind
+        self.pos_args = list(pos_args)
 
 
 class CompletionWidgetStub(QObject):
@@ -70,39 +68,45 @@ def completer_obj(qtbot, status_command_stub, config_stub, monkeypatch, stubs,
 
 
 @pytest.fixture(autouse=True)
-def instances(monkeypatch):
+def miscmodels_patch(mocker):
-    """Mock the instances module so get returns a fake completion model."""
+    """Patch the miscmodels module to provide fake completion functions.
-    # populate a model for each completion type, with a nested structure for
+
-    # option and value completion
+    Technically some of these are not part of miscmodels, but rolling them into
-    instances = {kind: FakeCompletionModel(kind)
+    one module is easier and sufficient for mocking. The only one referenced
-                 for kind in usertypes.Completion}
+    directly by Completer is miscmodels.command.
-    instances[usertypes.Completion.option] = {
+    """
-        'general': FakeCompletionModel(usertypes.Completion.option),
+    m = mocker.patch('qutebrowser.completion.completer.miscmodels',
-    }
+                     autospec=True)
-    instances[usertypes.Completion.value] = {
+    m.command = lambda *args: FakeCompletionModel('command', *args)
-        'general': {
+    m.helptopic = lambda *args: FakeCompletionModel('helptopic', *args)
-            'editor': FakeCompletionModel(usertypes.Completion.value),
+    m.quickmark = lambda *args: FakeCompletionModel('quickmark', *args)
-        }
+    m.bookmark = lambda *args: FakeCompletionModel('bookmark', *args)
-    }
+    m.session = lambda *args: FakeCompletionModel('session', *args)
-    monkeypatch.setattr(completer, 'instances', instances)
+    m.buffer = lambda *args: FakeCompletionModel('buffer', *args)
+    m.bind = lambda *args: FakeCompletionModel('bind', *args)
+    m.url = lambda *args: FakeCompletionModel('url', *args)
+    m.section = lambda *args: FakeCompletionModel('section', *args)
+    m.option = lambda *args: FakeCompletionModel('option', *args)
+    m.value = lambda *args: FakeCompletionModel('value', *args)
+    return m
 
 
 @pytest.fixture(autouse=True)
-def cmdutils_patch(monkeypatch, stubs):
+def cmdutils_patch(monkeypatch, stubs, miscmodels_patch):
     """Patch the cmdutils module to provide fake commands."""
-    @cmdutils.argument('section_', completion=usertypes.Completion.section)
+    @cmdutils.argument('section_', completion=miscmodels_patch.section)
-    @cmdutils.argument('option', completion=usertypes.Completion.option)
+    @cmdutils.argument('option', completion=miscmodels_patch.option)
-    @cmdutils.argument('value', completion=usertypes.Completion.value)
+    @cmdutils.argument('value', completion=miscmodels_patch.value)
     def set_command(section_=None, option=None, value=None):
         """docstring."""
         pass
 
-    @cmdutils.argument('topic', completion=usertypes.Completion.helptopic)
+    @cmdutils.argument('topic', completion=miscmodels_patch.helptopic)
     def show_help(tab=False, bg=False, window=False, topic=None):
         """docstring."""
         pass
 
-    @cmdutils.argument('url', completion=usertypes.Completion.url)
+    @cmdutils.argument('url', completion=miscmodels_patch.url)
     @cmdutils.argument('count', count=True)
     def openurl(url=None, implicit=False, bg=False, tab=False, window=False,
                 count=None):
@@ -110,7 +114,7 @@ def cmdutils_patch(monkeypatch, stubs):
         pass
 
     @cmdutils.argument('win_id', win_id=True)
-    @cmdutils.argument('command', completion=usertypes.Completion.command)
+    @cmdutils.argument('command', completion=miscmodels_patch.command)
     def bind(key, win_id, command=None, *, mode='normal', force=False):
         """docstring."""
         pass
@@ -140,60 +144,61 @@ def _set_cmd_prompt(cmd, txt):
     cmd.setCursorPosition(txt.index('|'))
 
 
-@pytest.mark.parametrize('txt, kind, pattern', [
+@pytest.mark.parametrize('txt, kind, pattern, pos_args', [
-    (':nope|', usertypes.Completion.command, 'nope'),
+    (':nope|', 'command', 'nope', []),
-    (':nope |', None, ''),
+    (':nope |', None, '', []),
-    (':set |', usertypes.Completion.section, ''),
+    (':set |', 'section', '', []),
-    (':set gen|', usertypes.Completion.section, 'gen'),
+    (':set gen|', 'section', 'gen', []),
-    (':set general |', usertypes.Completion.option, ''),
+    (':set general |', 'option', '', ['general']),
-    (':set what |', None, ''),
+    (':set what |', 'option', '', ['what']),
-    (':set general editor |', usertypes.Completion.value, ''),
+    (':set general editor |', 'value', '', ['general', 'editor']),
-    (':set general editor gv|', usertypes.Completion.value, 'gv'),
+    (':set general editor gv|', 'value', 'gv', ['general', 'editor']),
-    (':set general editor "gvim -f"|', usertypes.Completion.value, 'gvim -f'),
+    (':set general editor "gvim -f"|', 'value', 'gvim -f',
-    (':set general editor "gvim |', usertypes.Completion.value, 'gvim'),
+        ['general', 'editor']),
-    (':set general huh |', None, ''),
+    (':set general editor "gvim |', 'value', 'gvim', ['general', 'editor']),
-    (':help |', usertypes.Completion.helptopic, ''),
+    (':set general huh |', 'value', '', ['general', 'huh']),
-    (':help     |', usertypes.Completion.helptopic, ''),
+    (':help |', 'helptopic', '', []),
-    (':open |', usertypes.Completion.url, ''),
+    (':help     |', 'helptopic', '', []),
-    (':bind |', None, ''),
+    (':open |', 'url', '', []),
-    (':bind <c-x> |', usertypes.Completion.command, ''),
+    (':bind |', None, '', []),
-    (':bind <c-x> foo|', usertypes.Completion.command, 'foo'),
+    (':bind <c-x> |', 'command', '', ['<c-x>']),
-    (':bind <c-x>| foo', None, '<c-x>'),
+    (':bind <c-x> foo|', 'command', 'foo', ['<c-x>']),
-    (':set| general ', usertypes.Completion.command, 'set'),
+    (':bind <c-x>| foo', None, '<c-x>', []),
-    (':|set general ', usertypes.Completion.command, 'set'),
+    (':set| general ', 'command', 'set', []),
-    (':set gene|ral ignore-case', usertypes.Completion.section, 'general'),
+    (':|set general ', 'command', 'set', []),
-    (':|', usertypes.Completion.command, ''),
+    (':set gene|ral ignore-case', 'section', 'general', []),
-    (':   |', usertypes.Completion.command, ''),
+    (':|', 'command', '', []),
-    ('/|', None, ''),
+    (':   |', 'command', '', []),
-    (':open -t|', None, ''),
+    ('/|', None, '', []),
-    (':open --tab|', None, ''),
+    (':open -t|', None, '', []),
-    (':open -t |', usertypes.Completion.url, ''),
+    (':open --tab|', None, '', []),
-    (':open --tab |', usertypes.Completion.url, ''),
+    (':open -t |', 'url', '', []),
-    (':open | -t', usertypes.Completion.url, ''),
+    (':open --tab |', 'url', '', []),
-    (':tab-detach |', None, ''),
+    (':open | -t', 'url', '', []),
-    (':bind --mode=caret <c-x> |', usertypes.Completion.command, ''),
+    (':tab-detach |', None, '', []),
-    pytest.param(':bind --mode caret <c-x> |', usertypes.Completion.command,
+    (':bind --mode=caret <c-x> |', 'command', '', ['<c-x>']),
-                 '', marks=pytest.mark.xfail(reason='issue #74')),
+    pytest.param(':bind --mode caret <c-x> |', 'command', '', [],
-    (':set -t -p |', usertypes.Completion.section, ''),
+                 marks=pytest.mark.xfail(reason='issue #74')),
-    (':open -- |', None, ''),
+    (':set -t -p |', 'section', '', []),
-    (':gibberish nonesense |', None, ''),
+    (':open -- |', None, '', []),
-    ('/:help|', None, ''),
+    (':gibberish nonesense |', None, '', []),
-    ('::bind|', usertypes.Completion.command, ':bind'),
+    ('/:help|', None, '', []),
+    ('::bind|', 'command', ':bind', []),
 ])
-def test_update_completion(txt, kind, pattern, status_command_stub,
+def test_update_completion(txt, kind, pattern, pos_args, status_command_stub,
                            completer_obj, completion_widget_stub):
     """Test setting the completion widget's model based on command text."""
     # this test uses | as a placeholder for the current cursor position
     _set_cmd_prompt(status_command_stub, txt)
     completer_obj.schedule_completion_update()
-    assert completion_widget_stub.set_model.call_count == 1
-    args = completion_widget_stub.set_model.call_args[0]
-    # the outer model is just for sorting; srcmodel is the completion model
     if kind is None:
-        assert args[0] is None
+        assert completion_widget_stub.set_pattern.call_count == 0
     else:
-        assert args[0].srcmodel.kind == kind
+        assert completion_widget_stub.set_model.call_count == 1
-        assert args[1] == pattern
+        model = completion_widget_stub.set_model.call_args[0][0]
+        assert model.kind == kind
+        assert model.pos_args == pos_args
+        completion_widget_stub.set_pattern.assert_called_once_with(pattern)
 
 
 @pytest.mark.parametrize('before, newtxt, after', [

tests/unit/completion/test_completionmodel.py

@@ -0,0 +1,99 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Ryan Roden-Corrent (rcorre) <ryan@rcorre.net>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Tests for CompletionModel."""
+
+from unittest import mock
+import hypothesis
+from hypothesis import strategies
+
+import pytest
+from PyQt5.QtCore import QModelIndex
+
+from qutebrowser.completion.models import completionmodel, listcategory
+from qutebrowser.utils import qtutils
+
+
+@hypothesis.given(strategies.lists(min_size=0, max_size=3,
+    elements=strategies.integers(min_value=0, max_value=2**31)))
+def test_first_last_item(counts):
+    """Test that first() and last() index to the first and last items."""
+    model = completionmodel.CompletionModel()
+    for c in counts:
+        cat = mock.Mock(spec=['layoutChanged', 'layoutAboutToBeChanged'])
+        cat.rowCount = mock.Mock(return_value=c, spec=[])
+        model.add_category(cat)
+    data = [i for i, rowCount in enumerate(counts) if rowCount > 0]
+    if not data:
+        # with no items, first and last should be an invalid index
+        assert not model.first_item().isValid()
+        assert not model.last_item().isValid()
+    else:
+        first = data[0]
+        last = data[-1]
+        # first item of the first data category
+        assert model.first_item().row() == 0
+        assert model.first_item().parent().row() == first
+        # last item of the last data category
+        assert model.last_item().row() == counts[last] - 1
+        assert model.last_item().parent().row() == last
+
+
+@hypothesis.given(strategies.lists(elements=strategies.integers(),
+                                   min_size=0, max_size=3))
+def test_count(counts):
+    model = completionmodel.CompletionModel()
+    for c in counts:
+        cat = mock.Mock(spec=['rowCount', 'layoutChanged',
+                              'layoutAboutToBeChanged'])
+        cat.rowCount = mock.Mock(return_value=c, spec=[])
+        model.add_category(cat)
+    assert model.count() == sum(counts)
+
+
+@hypothesis.given(strategies.text())
+def test_set_pattern(pat):
+    """Validate the filtering and sorting results of set_pattern."""
+    model = completionmodel.CompletionModel()
+    cats = [mock.Mock(spec=['set_pattern', 'layoutChanged',
+                            'layoutAboutToBeChanged'])
+            for _ in range(3)]
+    for c in cats:
+        c.set_pattern = mock.Mock(spec=[])
+        model.add_category(c)
+    model.set_pattern(pat)
+    for c in cats:
+        c.set_pattern.assert_called_with(pat)
+
+
+def test_delete_cur_item():
+    func = mock.Mock(spec=[])
+    model = completionmodel.CompletionModel()
+    cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=func)
+    model.add_category(cat)
+    parent = model.index(0, 0)
+    model.delete_cur_item(model.index(0, 0, parent))
+    func.assert_called_once_with(['foo', 'bar'])
+
+
+def test_delete_cur_item_no_cat():
+    """Test completion_item_del with no selected category."""
+    model = completionmodel.CompletionModel()
+    with pytest.raises(qtutils.QtValueError):
+        model.delete_cur_item(QModelIndex())

tests/unit/completion/test_completionwidget.py

@@ -19,13 +19,14 @@
 
 """Tests for the CompletionView Object."""
 
-import unittest.mock
+from unittest import mock
 
 import pytest
-from PyQt5.QtGui import QStandardItem, QColor
+from PyQt5.QtGui import QColor
 
 from qutebrowser.completion import completionwidget
-from qutebrowser.completion.models import base, sortfilter
+from qutebrowser.completion.models import completionmodel, listcategory
+from qutebrowser.commands import cmdexc
 
 
 @pytest.fixture
@@ -71,23 +72,28 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry,
 
 def test_set_model(completionview):
     """Ensure set_model actually sets the model and expands all categories."""
-    model = base.BaseCompletionModel()
+    model = completionmodel.CompletionModel()
-    filtermodel = sortfilter.CompletionFilterModel(model)
     for i in range(3):
-        model.appendRow(QStandardItem(str(i)))
+        model.add_category(listcategory.ListCategory('', [('foo',)]))
-    completionview.set_model(filtermodel)
+    completionview.set_model(model)
-    assert completionview.model() is filtermodel
+    assert completionview.model() is model
-    for i in range(model.rowCount()):
+    for i in range(3):
-        assert completionview.isExpanded(filtermodel.index(i, 0))
+        assert completionview.isExpanded(model.index(i, 0))
 
 
 def test_set_pattern(completionview):
-    model = sortfilter.CompletionFilterModel(base.BaseCompletionModel())
+    model = completionmodel.CompletionModel()
-    model.set_pattern = unittest.mock.Mock()
+    model.set_pattern = mock.Mock(spec=[])
-    completionview.set_model(model, 'foo')
+    completionview.set_model(model)
+    completionview.set_pattern('foo')
     model.set_pattern.assert_called_with('foo')
 
 
+def test_set_pattern_no_model(completionview):
+    """Ensure that setting a pattern with no model does not fail."""
+    completionview.set_pattern('foo')
+
+
 def test_maybe_update_geometry(completionview, config_stub, qtbot):
     """Ensure completion is resized only if shrink is True."""
     with qtbot.assertNotEmitted(completionview.update_geometry):
@@ -148,15 +154,11 @@ def test_completion_item_focus(which, tree, expected, completionview, qtbot):
                   successive movement. None implies no signal should be
                   emitted.
     """
-    model = base.BaseCompletionModel()
+    model = completionmodel.CompletionModel()
     for catdata in tree:
-        cat = QStandardItem()
+        cat = listcategory.ListCategory('', ((x,) for x in catdata))
-        model.appendRow(cat)
+        model.add_category(cat)
-        for name in catdata:
+    completionview.set_model(model)
-            cat.appendRow(QStandardItem(name))
-    filtermodel = sortfilter.CompletionFilterModel(model,
-                                                   parent=completionview)
-    completionview.set_model(filtermodel)
     for entry in expected:
         if entry is None:
             with qtbot.assertNotEmitted(completionview.selection_changed):
@@ -176,10 +178,8 @@ def test_completion_item_focus_no_model(which, completionview, qtbot):
     """
     with qtbot.assertNotEmitted(completionview.selection_changed):
         completionview.completion_item_focus(which)
-    model = base.BaseCompletionModel()
+    model = completionmodel.CompletionModel()
-    filtermodel = sortfilter.CompletionFilterModel(model,
+    completionview.set_model(model)
-                                                   parent=completionview)
-    completionview.set_model(filtermodel)
     completionview.set_model(None)
     with qtbot.assertNotEmitted(completionview.selection_changed):
         completionview.completion_item_focus(which)
@@ -200,16 +200,13 @@ def test_completion_show(show, rows, quick_complete, completionview,
     config_stub.data['completion']['show'] = show
     config_stub.data['completion']['quick-complete'] = quick_complete
 
-    model = base.BaseCompletionModel()
+    model = completionmodel.CompletionModel()
     for name in rows:
-        cat = QStandardItem()
+        cat = listcategory.ListCategory('', [(name,)])
-        model.appendRow(cat)
+        model.add_category(cat)
-        cat.appendRow(QStandardItem(name))
-    filtermodel = sortfilter.CompletionFilterModel(model,
-                                                   parent=completionview)
 
     assert not completionview.isVisible()
-    completionview.set_model(filtermodel)
+    completionview.set_model(model)
     assert completionview.isVisible() == (show == 'always' and len(rows) > 0)
     completionview.completion_item_focus('next')
     expected = (show != 'never' and len(rows) > 0 and
@@ -218,3 +215,27 @@ def test_completion_show(show, rows, quick_complete, completionview,
     completionview.set_model(None)
     completionview.completion_item_focus('next')
     assert not completionview.isVisible()
+
+
+def test_completion_item_del(completionview):
+    """Test that completion_item_del invokes delete_cur_item in the model."""
+    func = mock.Mock(spec=[])
+    model = completionmodel.CompletionModel()
+    cat = listcategory.ListCategory('', [('foo', 'bar')], delete_func=func)
+    model.add_category(cat)
+    completionview.set_model(model)
+    completionview.completion_item_focus('next')
+    completionview.completion_item_del()
+    func.assert_called_once_with(['foo', 'bar'])
+
+
+def test_completion_item_del_no_selection(completionview):
+    """Test that completion_item_del with an invalid index."""
+    func = mock.Mock(spec=[])
+    model = completionmodel.CompletionModel()
+    cat = listcategory.ListCategory('', [('foo',)], delete_func=func)
+    model.add_category(cat)
+    completionview.set_model(model)
+    with pytest.raises(cmdexc.CommandError, match='No item selected!'):
+        completionview.completion_item_del()
+    assert not func.called

tests/unit/completion/test_histcategory.py

@@ -0,0 +1,150 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) <ryan@rcorre.net>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the web history completion category."""
+
+import unittest.mock
+import datetime
+
+import pytest
+
+from qutebrowser.misc import sql
+from qutebrowser.completion.models import histcategory
+from qutebrowser.commands import cmdexc
+
+
+@pytest.fixture
+def hist(init_sql, config_stub):
+    config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d',
+                                      'web-history-max-items': -1}
+    return sql.SqlTable('CompletionHistory', ['url', 'title', 'last_atime'])
+
+
+@pytest.mark.parametrize('pattern, before, after', [
+    ('foo',
+     [('foo', ''), ('bar', ''), ('aafobbb', '')],
+     [('foo',)]),
+
+    ('FOO',
+     [('foo', ''), ('bar', ''), ('aafobbb', '')],
+     [('foo',)]),
+
+    ('foo',
+     [('FOO', ''), ('BAR', ''), ('AAFOBBB', '')],
+     [('FOO',)]),
+
+    ('foo',
+     [('baz', 'bar'), ('foo', ''), ('bar', 'foo')],
+     [('foo', ''), ('bar', 'foo')]),
+
+    ('foo',
+     [('fooa', ''), ('foob', ''), ('fooc', '')],
+     [('fooa', ''), ('foob', ''), ('fooc', '')]),
+
+    ('foo',
+     [('foo', 'bar'), ('bar', 'foo'), ('biz', 'baz')],
+     [('foo', 'bar'), ('bar', 'foo')]),
+
+    ('foo bar',
+     [('foo', ''), ('bar foo', ''), ('xfooyybarz', '')],
+     [('xfooyybarz', '')]),
+
+    ('foo%bar',
+     [('foo%bar', ''), ('foo bar', ''), ('foobar', '')],
+     [('foo%bar', '')]),
+
+    ('_',
+     [('a_b', ''), ('__a', ''), ('abc', '')],
+     [('a_b', ''), ('__a', '')]),
+
+    ('%',
+     [('\\foo', '\\bar')],
+     []),
+
+    ("can't",
+     [("can't touch this", ''), ('a', '')],
+     [("can't touch this", '')]),
+])
+def test_set_pattern(pattern, before, after, model_validator, hist):
+    """Validate the filtering and sorting results of set_pattern."""
+    for row in before:
+        hist.insert({'url': row[0], 'title': row[1], 'last_atime': 1})
+    cat = histcategory.HistoryCategory()
+    model_validator.set_model(cat)
+    cat.set_pattern(pattern)
+    model_validator.validate(after)
+
+
+@pytest.mark.parametrize('max_items, before, after', [
+    (-1, [
+        ('a', 'a', '2017-04-16'),
+        ('b', 'b', '2017-06-16'),
+        ('c', 'c', '2017-05-16'),
+    ], [
+        ('b', 'b', '2017-06-16'),
+        ('c', 'c', '2017-05-16'),
+        ('a', 'a', '2017-04-16'),
+    ]),
+    (3, [
+        ('a', 'a', '2017-04-16'),
+        ('b', 'b', '2017-06-16'),
+        ('c', 'c', '2017-05-16'),
+    ], [
+        ('b', 'b', '2017-06-16'),
+        ('c', 'c', '2017-05-16'),
+        ('a', 'a', '2017-04-16'),
+    ]),
+    (2, [
+        ('a', 'a', '2017-04-16'),
+        ('b', 'b', '2017-06-16'),
+        ('c', 'c', '2017-05-16'),
+    ], [
+        ('b', 'b', '2017-06-16'),
+        ('c', 'c', '2017-05-16'),
+    ])
+])
+def test_sorting(max_items, before, after, model_validator, hist, config_stub):
+    """Validate the filtering and sorting results of set_pattern."""
+    config_stub.data['completion']['web-history-max-items'] = max_items
+    for url, title, atime in before:
+        timestamp = datetime.datetime.strptime(atime, '%Y-%m-%d').timestamp()
+        hist.insert({'url': url, 'title': title, 'last_atime': timestamp})
+    cat = histcategory.HistoryCategory()
+    model_validator.set_model(cat)
+    cat.set_pattern('')
+    model_validator.validate(after)
+
+
+def test_delete_cur_item(hist):
+    hist.insert({'url': 'foo', 'title': 'Foo'})
+    hist.insert({'url': 'bar', 'title': 'Bar'})
+    func = unittest.mock.Mock(spec=[])
+    cat = histcategory.HistoryCategory(delete_func=func)
+    cat.set_pattern('')
+    cat.delete_cur_item(cat.index(0, 0))
+    func.assert_called_with(['foo', 'Foo', ''])
+
+
+def test_delete_cur_item_no_func(hist):
+    hist.insert({'url': 'foo', 'title': 1})
+    hist.insert({'url': 'bar', 'title': 2})
+    cat = histcategory.HistoryCategory()
+    cat.set_pattern('')
+    with pytest.raises(cmdexc.CommandError, match='Cannot delete this item'):
+        cat.delete_cur_item(cat.index(0, 0))

tests/unit/completion/test_listcategory.py

@@ -0,0 +1,73 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Ryan Roden-Corrent (rcorre) <ryan@rcorre.net>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Tests for CompletionFilterModel."""
+
+from unittest import mock
+
+import pytest
+
+from qutebrowser.completion.models import listcategory
+from qutebrowser.commands import cmdexc
+
+
+@pytest.mark.parametrize('pattern, before, after', [
+    ('foo',
+     [('foo', ''), ('bar', '')],
+     [('foo', '')]),
+
+    ('foo',
+     [('foob', ''), ('fooc', ''), ('fooa', '')],
+     [('fooa', ''), ('foob', ''), ('fooc', '')]),
+
+    # prefer foobar as it starts with the pattern
+    ('foo',
+     [('barfoo', ''), ('foobar', '')],
+     [('foobar', ''), ('barfoo', '')]),
+
+    ('foo',
+     [('foo', 'bar'), ('bar', 'foo'), ('bar', 'bar')],
+     [('foo', 'bar'), ('bar', 'foo')]),
+])
+def test_set_pattern(pattern, before, after, model_validator):
+    """Validate the filtering and sorting results of set_pattern."""
+    cat = listcategory.ListCategory('Foo', before)
+    model_validator.set_model(cat)
+    cat.set_pattern(pattern)
+    model_validator.validate(after)
+
+
+def test_delete_cur_item(model_validator):
+    func = mock.Mock(spec=[])
+    cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')],
+                                    delete_func=func)
+    model_validator.set_model(cat)
+    idx = cat.index(0, 0)
+    cat.delete_cur_item(idx)
+    func.assert_called_once_with(['a', 'b'])
+    model_validator.validate([('c', 'd')])
+
+
+def test_delete_cur_item_no_func(model_validator):
+    cat = listcategory.ListCategory('Foo', [('a', 'b'), ('c', 'd')])
+    model_validator.set_model(cat)
+    idx = cat.index(0, 0)
+    with pytest.raises(cmdexc.CommandError, match="Cannot delete this item."):
+        cat.delete_cur_item(idx)
+    model_validator.validate([('a', 'b'), ('c', 'd')])

tests/unit/completion/test_models.py

@@ -24,12 +24,11 @@ from datetime import datetime
 
 import pytest
 from PyQt5.QtCore import QUrl
-from PyQt5.QtWidgets import QTreeView
 
-from qutebrowser.completion.models import (miscmodels, urlmodel, configmodel,
+from qutebrowser.completion.models import miscmodels, urlmodel, configmodel
-                                           sortfilter)
-from qutebrowser.browser import history
 from qutebrowser.config import sections, value
+from qutebrowser.utils import objreg
+from qutebrowser.browser import history
 
 
 def _check_completions(model, expected):
@@ -43,19 +42,21 @@ def _check_completions(model, expected):
                 ...
             }
     """
+    actual = {}
     assert model.rowCount() == len(expected)
     for i in range(0, model.rowCount()):
-        actual_cat = model.item(i)
+        catidx = model.index(i, 0)
-        catname = actual_cat.text()
+        catname = model.data(catidx)
-        assert catname in expected
+        actual[catname] = []
-        expected_cat = expected[catname]
+        for j in range(model.rowCount(catidx)):
-        assert actual_cat.rowCount() == len(expected_cat)
+            name = model.data(model.index(j, 0, parent=catidx))
-        for j in range(0, actual_cat.rowCount()):
+            desc = model.data(model.index(j, 1, parent=catidx))
-            name = actual_cat.child(j, 0)
+            misc = model.data(model.index(j, 2, parent=catidx))
-            desc = actual_cat.child(j, 1)
+            actual[catname].append((name, desc, misc))
-            misc = actual_cat.child(j, 2)
+    assert actual == expected
-            actual_item = (name.text(), desc.text(), misc.text())
+    # sanity-check the column_widths
-            assert actual_item in expected_cat
+    assert len(model.column_widths) == 3
+    assert sum(model.column_widths) == 100
 
 
 def _patch_cmdutils(monkeypatch, stubs, symbol):
@@ -113,22 +114,6 @@ def _patch_config_section_desc(monkeypatch, stubs, symbol):
     monkeypatch.setattr(symbol, section_desc)
 
 
-def _mock_view_index(model, category_idx, child_idx, qtbot):
-    """Create a tree view from a model and set the current index.
-
-    Args:
-        model: model to create a fake view for.
-        category_idx: index of the category to select.
-        child_idx: index of the child item under that category to select.
-    """
-    view = QTreeView()
-    qtbot.add_widget(view)
-    view.setModel(model)
-    idx = model.indexFromItem(model.item(category_idx).child(child_idx))
-    view.setCurrentIndex(idx)
-    return view
-
-
 @pytest.fixture
 def quickmarks(quickmark_manager_stub):
     """Pre-populate the quickmark-manager stub with some quickmarks."""
@@ -152,20 +137,35 @@ def bookmarks(bookmark_manager_stub):
 
 
 @pytest.fixture
-def web_history(stubs, web_history_stub):
+def web_history(init_sql, stubs, config_stub):
-    """Pre-populate the web-history stub with some history entries."""
+    """Fixture which provides a web-history object."""
-    web_history_stub.history_dict = collections.OrderedDict([
+    config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d',
-        ('http://qutebrowser.org', history.Entry(
+                                      'web-history-max-items': -1}
-            datetime(2015, 9, 5).timestamp(),
+    stub = history.WebHistory()
-            QUrl('http://qutebrowser.org'), 'qutebrowser | qutebrowser')),
+    objreg.register('web-history', stub)
-        ('https://python.org', history.Entry(
+    yield stub
-            datetime(2016, 3, 8).timestamp(),
+    objreg.delete('web-history')
-            QUrl('https://python.org'), 'Welcome to Python.org')),
+
-        ('https://github.com', history.Entry(
+
-            datetime(2016, 5, 1).timestamp(),
+@pytest.fixture
-            QUrl('https://github.com'), 'GitHub')),
+def web_history_populated(web_history):
-    ])
+    """Pre-populate the web-history database."""
-    return web_history_stub
+    web_history.add_url(
+        url=QUrl('http://qutebrowser.org'),
+        title='qutebrowser',
+        atime=datetime(2015, 9, 5).timestamp()
+    )
+    web_history.add_url(
+        url=QUrl('https://python.org'),
+        title='Welcome to Python.org',
+        atime=datetime(2016, 3, 8).timestamp()
+    )
+    web_history.add_url(
+        url=QUrl('https://github.com'),
+        title='https://github.com',
+        atime=datetime(2016, 5, 1).timestamp()
+    )
+    return web_history
 
 
 def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub,
@@ -184,16 +184,17 @@ def test_command_completion(qtmodeltester, monkeypatch, stubs, config_stub,
     key_config_stub.set_bindings_for('normal', {'s': 'stop',
                                                 'rr': 'roll',
                                                 'ro': 'rock'})
-    model = miscmodels.CommandCompletionModel()
+    model = miscmodels.command()
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
     _check_completions(model, {
         "Commands": [
-            ('stop', 'stop qutebrowser', 's'),
             ('drop', 'drop all user data', ''),
-            ('roll', 'never gonna give you up', 'rr'),
             ('rock', "Alias for 'roll'", 'ro'),
+            ('roll', 'never gonna give you up', 'rr'),
+            ('stop', 'stop qutebrowser', 's'),
         ]
     })
 
@@ -212,135 +213,200 @@ def test_help_completion(qtmodeltester, monkeypatch, stubs, key_config_stub):
     key_config_stub.set_bindings_for('normal', {'s': 'stop', 'rr': 'roll'})
     _patch_cmdutils(monkeypatch, stubs, module + '.cmdutils')
     _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA')
-    model = miscmodels.HelpCompletionModel()
+    model = miscmodels.helptopic()
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
     _check_completions(model, {
         "Commands": [
-            (':stop', 'stop qutebrowser', 's'),
             (':drop', 'drop all user data', ''),
-            (':roll', 'never gonna give you up', 'rr'),
             (':hide', '', ''),
+            (':roll', 'never gonna give you up', 'rr'),
+            (':stop', 'stop qutebrowser', 's'),
         ],
         "Settings": [
-            ('general->time', 'Is an illusion.', ''),
+            ('general->time', 'Is an illusion.', None),
-            ('general->volume', 'Goes to 11', ''),
+            ('general->volume', 'Goes to 11', None),
-            ('ui->gesture', 'Waggle your hands to control qutebrowser', ''),
+            ('searchengines->DEFAULT', '', None),
-            ('ui->mind', 'Enable mind-control ui (experimental)', ''),
+            ('ui->gesture', 'Waggle your hands to control qutebrowser', None),
-            ('ui->voice', 'Whether to respond to voice commands', ''),
+            ('ui->mind', 'Enable mind-control ui (experimental)', None),
-            ('searchengines->DEFAULT', '', ''),
+            ('ui->voice', 'Whether to respond to voice commands', None),
         ]
     })
 
 
 def test_quickmark_completion(qtmodeltester, quickmarks):
     """Test the results of quickmark completion."""
-    model = miscmodels.QuickmarkCompletionModel()
+    model = miscmodels.quickmark()
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
     _check_completions(model, {
         "Quickmarks": [
-            ('aw', 'https://wiki.archlinux.org', ''),
+            ('aw', 'https://wiki.archlinux.org', None),
-            ('ddg', 'https://duckduckgo.com', ''),
+            ('ddg', 'https://duckduckgo.com', None),
-            ('wiki', 'https://wikipedia.org', ''),
+            ('wiki', 'https://wikipedia.org', None),
         ]
     })
 
 
 def test_bookmark_completion(qtmodeltester, bookmarks):
     """Test the results of bookmark completion."""
-    model = miscmodels.BookmarkCompletionModel()
+    model = miscmodels.bookmark()
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
     _check_completions(model, {
         "Bookmarks": [
-            ('https://github.com', 'GitHub', ''),
+            ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None),
-            ('https://python.org', 'Welcome to Python.org', ''),
+            ('https://github.com', 'GitHub', None),
-            ('http://qutebrowser.org', 'qutebrowser | qutebrowser', ''),
+            ('https://python.org', 'Welcome to Python.org', None),
         ]
     })
 
 
-def test_url_completion(qtmodeltester, config_stub, web_history, quickmarks,
+def test_url_completion(qtmodeltester, web_history_populated,
-                        bookmarks):
+                        quickmarks, bookmarks):
     """Test the results of url completion.
 
     Verify that:
         - quickmarks, bookmarks, and urls are included
-        - no more than 'web-history-max-items' history entries are included
+        - entries are sorted by access time
-        - the most recent entries are included
+        - only the most recent entry is included for each url
     """
-    config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d',
+    model = urlmodel.url()
-                                      'web-history-max-items': 2}
+    model.set_pattern('')
-    model = urlmodel.UrlCompletionModel()
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
     _check_completions(model, {
         "Quickmarks": [
-            ('https://wiki.archlinux.org', 'aw', ''),
+            ('https://duckduckgo.com', 'ddg', None),
-            ('https://duckduckgo.com', 'ddg', ''),
+            ('https://wiki.archlinux.org', 'aw', None),
-            ('https://wikipedia.org', 'wiki', ''),
+            ('https://wikipedia.org', 'wiki', None),
         ],
         "Bookmarks": [
-            ('https://github.com', 'GitHub', ''),
+            ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None),
-            ('https://python.org', 'Welcome to Python.org', ''),
+            ('https://github.com', 'GitHub', None),
-            ('http://qutebrowser.org', 'qutebrowser | qutebrowser', ''),
+            ('https://python.org', 'Welcome to Python.org', None),
         ],
         "History": [
+            ('https://github.com', 'https://github.com', '2016-05-01'),
             ('https://python.org', 'Welcome to Python.org', '2016-03-08'),
-            ('https://github.com', 'GitHub', '2016-05-01'),
+            ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'),
         ],
     })
 
 
-def test_url_completion_delete_bookmark(qtmodeltester, config_stub,
+@pytest.mark.parametrize('url, title, pattern, rowcount', [
-                                        web_history, quickmarks, bookmarks,
+    ('example.com', 'Site Title', '', 1),
-                                        qtbot):
+    ('example.com', 'Site Title', 'ex', 1),
+    ('example.com', 'Site Title', 'am', 1),
+    ('example.com', 'Site Title', 'com', 1),
+    ('example.com', 'Site Title', 'ex com', 1),
+    ('example.com', 'Site Title', 'com ex', 0),
+    ('example.com', 'Site Title', 'ex foo', 0),
+    ('example.com', 'Site Title', 'foo com', 0),
+    ('example.com', 'Site Title', 'exm', 0),
+    ('example.com', 'Site Title', 'Si Ti', 1),
+    ('example.com', 'Site Title', 'Ti Si', 0),
+    ('example.com', '', 'foo', 0),
+    ('foo_bar', '', '_', 1),
+    ('foobar', '', '_', 0),
+    ('foo%bar', '', '%', 1),
+    ('foobar', '', '%', 0),
+])
+def test_url_completion_pattern(web_history, quickmark_manager_stub,
+                                bookmark_manager_stub, url, title, pattern,
+                                rowcount):
+    """Test that url completion filters by url and title."""
+    web_history.add_url(QUrl(url), title)
+    model = urlmodel.url()
+    model.set_pattern(pattern)
+    # 2, 0 is History
+    assert model.rowCount(model.index(2, 0)) == rowcount
+
+
+def test_url_completion_delete_bookmark(qtmodeltester, bookmarks,
+                                        web_history, quickmarks):
     """Test deleting a bookmark from the url completion model."""
-    config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d',
+    model = urlmodel.url()
-                                      'web-history-max-items': 2}
+    model.set_pattern('')
-    model = urlmodel.UrlCompletionModel()
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
-    # delete item (1, 0) -> (bookmarks, 'https://github.com' )
+    parent = model.index(1, 0)
-    view = _mock_view_index(model, 1, 0, qtbot)
+    idx = model.index(1, 0, parent)
-    model.delete_cur_item(view)
+
+    # sanity checks
+    assert model.data(parent) == "Bookmarks"
+    assert model.data(idx) == 'https://github.com'
+    assert 'https://github.com' in bookmarks.marks
+
+    len_before = len(bookmarks.marks)
+    model.delete_cur_item(idx)
     assert 'https://github.com' not in bookmarks.marks
-    assert 'https://python.org' in bookmarks.marks
+    assert len_before == len(bookmarks.marks) + 1
-    assert 'http://qutebrowser.org' in bookmarks.marks
 
 
-def test_url_completion_delete_quickmark(qtmodeltester, config_stub,
+def test_url_completion_delete_quickmark(qtmodeltester,
-                                         web_history, quickmarks, bookmarks,
+                                         quickmarks, web_history, bookmarks,
                                          qtbot):
     """Test deleting a bookmark from the url completion model."""
-    config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d',
+    model = urlmodel.url()
-                                      'web-history-max-items': 2}
+    model.set_pattern('')
-    model = urlmodel.UrlCompletionModel()
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
-    # delete item (0, 1) -> (quickmarks, 'ddg' )
+    parent = model.index(0, 0)
-    view = _mock_view_index(model, 0, 1, qtbot)
+    idx = model.index(0, 0, parent)
-    model.delete_cur_item(view)
+
-    assert 'aw' in quickmarks.marks
+    # sanity checks
+    assert model.data(parent) == "Quickmarks"
+    assert model.data(idx) == 'https://duckduckgo.com'
+    assert 'ddg' in quickmarks.marks
+
+    len_before = len(quickmarks.marks)
+    model.delete_cur_item(idx)
     assert 'ddg' not in quickmarks.marks
-    assert 'wiki' in quickmarks.marks
+    assert len_before == len(quickmarks.marks) + 1
+
+
+def test_url_completion_delete_history(qtmodeltester,
+                                       web_history_populated,
+                                       quickmarks, bookmarks):
+    """Test deleting a history entry."""
+    model = urlmodel.url()
+    model.set_pattern('')
+    qtmodeltester.data_display_may_return_none = True
+    qtmodeltester.check(model)
+
+    parent = model.index(2, 0)
+    idx = model.index(1, 0, parent)
+
+    # sanity checks
+    assert model.data(parent) == "History"
+    assert model.data(idx) == 'https://python.org'
+
+    assert 'https://python.org' in web_history_populated
+    model.delete_cur_item(idx)
+    assert 'https://python.org' not in web_history_populated
 
 
 def test_session_completion(qtmodeltester, session_manager_stub):
     session_manager_stub.sessions = ['default', '1', '2']
-    model = miscmodels.SessionCompletionModel()
+    model = miscmodels.session()
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
     _check_completions(model, {
-        "Sessions": [('default', '', ''), ('1', '', ''), ('2', '', '')]
+        "Sessions": [('1', None, None),
+                     ('2', None, None),
+                     ('default', None, None)]
     })
 
 
@@ -354,7 +420,8 @@ def test_tab_completion(qtmodeltester, fake_web_tab, app_stub, win_registry,
     tabbed_browser_stubs[1].tabs = [
         fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0),
     ]
-    model = miscmodels.TabCompletionModel()
+    model = miscmodels.buffer()
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
@@ -370,7 +437,7 @@ def test_tab_completion(qtmodeltester, fake_web_tab, app_stub, win_registry,
     })
 
 
-def test_tab_completion_delete(qtmodeltester, fake_web_tab, qtbot, app_stub,
+def test_tab_completion_delete(qtmodeltester, fake_web_tab, app_stub,
                                win_registry, tabbed_browser_stubs):
     """Verify closing a tab by deleting it from the completion widget."""
     tabbed_browser_stubs[0].tabs = [
@@ -381,13 +448,19 @@ def test_tab_completion_delete(qtmodeltester, fake_web_tab, qtbot, app_stub,
     tabbed_browser_stubs[1].tabs = [
         fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0),
     ]
-    model = miscmodels.TabCompletionModel()
+    model = miscmodels.buffer()
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
-    view = _mock_view_index(model, 0, 1, qtbot)
+    parent = model.index(0, 0)
-    qtbot.add_widget(view)
+    idx = model.index(1, 0, parent)
-    model.delete_cur_item(view)
+
+    # sanity checks
+    assert model.data(parent) == "0"
+    assert model.data(idx) == '0/2'
+
+    model.delete_cur_item(idx)
     actual = [tab.url() for tab in tabbed_browser_stubs[0].tabs]
     assert actual == [QUrl('https://github.com'),
                       QUrl('https://duckduckgo.com')]
@@ -398,15 +471,16 @@ def test_setting_section_completion(qtmodeltester, monkeypatch, stubs):
     _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA')
     _patch_config_section_desc(monkeypatch, stubs,
                                module + '.configdata.SECTION_DESC')
-    model = configmodel.SettingSectionCompletionModel()
+    model = configmodel.section()
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
     _check_completions(model, {
         "Sections": [
-            ('general', 'General/miscellaneous options.', ''),
+            ('general', 'General/miscellaneous options.', None),
-            ('ui', 'General options related to the user interface.', ''),
+            ('searchengines', 'Definitions of search engines ...', None),
-            ('searchengines', 'Definitions of search engines ...', ''),
+            ('ui', 'General options related to the user interface.', None),
         ]
     })
 
@@ -418,7 +492,8 @@ def test_setting_option_completion(qtmodeltester, monkeypatch, stubs,
     config_stub.data = {'ui': {'gesture': 'off',
                                'mind': 'on',
                                'voice': 'sometimes'}}
-    model = configmodel.SettingOptionCompletionModel('ui')
+    model = configmodel.option('ui')
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
@@ -431,6 +506,12 @@ def test_setting_option_completion(qtmodeltester, monkeypatch, stubs,
     })
 
 
+def test_setting_option_completion_empty(monkeypatch, stubs, config_stub):
+    module = 'qutebrowser.completion.models.configmodel'
+    _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA')
+    assert configmodel.option('typo') is None
+
+
 def test_setting_option_completion_valuelist(qtmodeltester, monkeypatch, stubs,
                                              config_stub):
     module = 'qutebrowser.completion.models.configmodel'
@@ -440,7 +521,8 @@ def test_setting_option_completion_valuelist(qtmodeltester, monkeypatch, stubs,
             'DEFAULT': 'https://duckduckgo.com/?q={}'
         }
     }
-    model = configmodel.SettingOptionCompletionModel('searchengines')
+    model = configmodel.option('searchengines')
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
@@ -454,22 +536,30 @@ def test_setting_value_completion(qtmodeltester, monkeypatch, stubs,
     module = 'qutebrowser.completion.models.configmodel'
     _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA')
     config_stub.data = {'general': {'volume': '0'}}
-    model = configmodel.SettingValueCompletionModel('general', 'volume')
+    model = configmodel.value('general', 'volume')
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
     _check_completions(model, {
         "Current/Default": [
-            ('0', 'Current value', ''),
+            ('0', 'Current value', None),
-            ('11', 'Default value', ''),
+            ('11', 'Default value', None),
         ],
         "Completions": [
-            ('0', '', ''),
+            ('0', '', None),
-            ('11', '', ''),
+            ('11', '', None),
         ]
     })
 
 
+def test_setting_value_completion_empty(monkeypatch, stubs, config_stub):
+    module = 'qutebrowser.completion.models.configmodel'
+    _patch_configdata(monkeypatch, stubs, module + '.configdata.DATA')
+    config_stub.data = {'general': {}}
+    assert configmodel.value('general', 'typo') is None
+
+
 def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub,
                          key_config_stub):
     """Test the results of keybinding command completion.
@@ -486,55 +576,55 @@ def test_bind_completion(qtmodeltester, monkeypatch, stubs, config_stub,
     key_config_stub.set_bindings_for('normal', {'s': 'stop',
                                                 'rr': 'roll',
                                                 'ro': 'rock'})
-    model = miscmodels.BindCompletionModel()
+    model = miscmodels.bind('s')
+    model.set_pattern('')
     qtmodeltester.data_display_may_return_none = True
     qtmodeltester.check(model)
 
     _check_completions(model, {
-        "Commands": [
+        "Current": [
             ('stop', 'stop qutebrowser', 's'),
+        ],
+        "Commands": [
             ('drop', 'drop all user data', ''),
             ('hide', '', ''),
-            ('roll', 'never gonna give you up', 'rr'),
             ('rock', "Alias for 'roll'", 'ro'),
+            ('roll', 'never gonna give you up', 'rr'),
+            ('stop', 'stop qutebrowser', 's'),
         ]
     })
 
 
-def test_url_completion_benchmark(benchmark, config_stub,
+def test_url_completion_benchmark(benchmark,
                                   quickmark_manager_stub,
                                   bookmark_manager_stub,
-                                  web_history_stub):
+                                  web_history):
     """Benchmark url completion."""
-    config_stub.data['completion'] = {'timestamp-format': '%Y-%m-%d',
+    r = range(100000)
-                                      'web-history-max-items': 1000}
+    entries = {
+        'last_atime': list(r),
+        'url': ['http://example.com/{}'.format(i) for i in r],
+        'title': ['title{}'.format(i) for i in r]
+    }
 
-    entries = [history.Entry(
+    web_history.completion.insert_batch(entries)
-        atime=i,
-        url=QUrl('http://example.com/{}'.format(i)),
-        title='title{}'.format(i))
-        for i in range(100000)]
 
-    web_history_stub.history_dict = collections.OrderedDict(
+    quickmark_manager_stub.marks = collections.OrderedDict([
-        ((e.url_str(), e) for e in entries))
+        ('title{}'.format(i), 'example.com/{}'.format(i))
+        for i in range(1000)])
 
-    quickmark_manager_stub.marks = collections.OrderedDict(
+    bookmark_manager_stub.marks = collections.OrderedDict([
-        (e.title, e.url_str())
+        ('example.com/{}'.format(i), 'title{}'.format(i))
-        for e in entries[0:1000])
+        for i in range(1000)])
-
-    bookmark_manager_stub.marks = collections.OrderedDict(
-        (e.url_str(), e.title)
-        for e in entries[0:1000])
 
     def bench():
-        model = urlmodel.UrlCompletionModel()
+        model = urlmodel.url()
-        filtermodel = sortfilter.CompletionFilterModel(model)
+        model.set_pattern('')
-        filtermodel.set_pattern('')
+        model.set_pattern('e')
-        filtermodel.set_pattern('e')
+        model.set_pattern('ex')
-        filtermodel.set_pattern('ex')
+        model.set_pattern('ex ')
-        filtermodel.set_pattern('ex ')
+        model.set_pattern('ex 1')
-        filtermodel.set_pattern('ex 1')
+        model.set_pattern('ex 12')
-        filtermodel.set_pattern('ex 12')
+        model.set_pattern('ex 123')
-        filtermodel.set_pattern('ex 123')
 
     benchmark(bench)

tests/unit/completion/test_sortfilter.py

@@ -1,230 +0,0 @@
-# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
-
-# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Tests for CompletionFilterModel."""
-
-import pytest
-
-from PyQt5.QtCore import Qt
-
-from qutebrowser.completion.models import base, sortfilter
-
-
-def _create_model(data):
-    """Create a completion model populated with the given data.
-
-    data: A list of lists, where each sub-list represents a category, each
-          tuple in the sub-list represents an item, and each value in the
-          tuple represents the item data for that column
-    """
-    model = base.BaseCompletionModel()
-    for catdata in data:
-        cat = model.new_category('')
-        for itemdata in catdata:
-            model.new_item(cat, *itemdata)
-    return model
-
-
-def _extract_model_data(model):
-    """Express a model's data as a list for easier comparison.
-
-    Return: A list of lists, where each sub-list represents a category, each
-            tuple in the sub-list represents an item, and each value in the
-            tuple represents the item data for that column
-    """
-    data = []
-    for i in range(0, model.rowCount()):
-        cat_idx = model.index(i, 0)
-        row = []
-        for j in range(0, model.rowCount(cat_idx)):
-            row.append((model.data(cat_idx.child(j, 0)),
-                        model.data(cat_idx.child(j, 1)),
-                        model.data(cat_idx.child(j, 2))))
-        data.append(row)
-    return data
-
-
-@pytest.mark.parametrize('pattern, data, expected', [
-    ('foo', 'barfoobar', True),
-    ('foo bar', 'barfoobar', True),
-    ('foo   bar', 'barfoobar', True),
-    ('foo bar', 'barfoobazbar', True),
-    ('foo   bar', 'barfoobazbar', True),
-    ('foo', 'barFOObar', True),
-    ('Foo', 'barfOObar', True),
-    ('ab', 'aonebtwo', False),
-    ('33', 'l33t', True),
-    ('x', 'blah', False),
-    ('4', 'blah', False),
-])
-def test_filter_accepts_row(pattern, data, expected):
-    source_model = base.BaseCompletionModel()
-    cat = source_model.new_category('test')
-    source_model.new_item(cat, data)
-
-    filter_model = sortfilter.CompletionFilterModel(source_model)
-    filter_model.set_pattern(pattern)
-    assert filter_model.rowCount() == 1  # "test" category
-    idx = filter_model.index(0, 0)
-    assert idx.isValid()
-
-    row_count = filter_model.rowCount(idx)
-    assert row_count == (1 if expected else 0)
-
-
-@pytest.mark.parametrize('tree, first, last', [
-    ([[('Aa',)]], 'Aa', 'Aa'),
-    ([[('Aa',)], [('Ba',)]], 'Aa', 'Ba'),
-    ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]],
-     'Aa', 'Ca'),
-    ([[], [('Ba',)]], 'Ba', 'Ba'),
-    ([[], [], [('Ca',)]], 'Ca', 'Ca'),
-    ([[], [], [('Ca',), ('Cb',)]], 'Ca', 'Cb'),
-    ([[('Aa',)], []], 'Aa', 'Aa'),
-    ([[('Aa',)], []], 'Aa', 'Aa'),
-    ([[('Aa',)], [], []], 'Aa', 'Aa'),
-    ([[('Aa',)], [], [('Ca',)]], 'Aa', 'Ca'),
-    ([[], []], None, None),
-])
-def test_first_last_item(tree, first, last):
-    """Test that first() and last() return indexes to the first and last items.
-
-    Args:
-        tree: Each list represents a completion category, with each string
-              being an item under that category.
-        first: text of the first item
-        last: text of the last item
-    """
-    model = _create_model(tree)
-    filter_model = sortfilter.CompletionFilterModel(model)
-    assert filter_model.data(filter_model.first_item()) == first
-    assert filter_model.data(filter_model.last_item()) == last
-
-
-def test_set_source_model():
-    """Ensure setSourceModel sets source_model and clears the pattern."""
-    model1 = base.BaseCompletionModel()
-    model2 = base.BaseCompletionModel()
-    filter_model = sortfilter.CompletionFilterModel(model1)
-    filter_model.set_pattern('foo')
-    # sourceModel() is cached as srcmodel, so make sure both match
-    assert filter_model.srcmodel is model1
-    assert filter_model.sourceModel() is model1
-    assert filter_model.pattern == 'foo'
-    filter_model.setSourceModel(model2)
-    assert filter_model.srcmodel is model2
-    assert filter_model.sourceModel() is model2
-    assert not filter_model.pattern
-
-
-@pytest.mark.parametrize('tree, expected', [
-    ([[('Aa',)]], 1),
-    ([[('Aa',)], [('Ba',)]], 2),
-    ([[('Aa',), ('Ab',), ('Ac',)], [('Ba',), ('Bb',)], [('Ca',)]], 6),
-    ([[], [('Ba',)]], 1),
-    ([[], [], [('Ca',)]], 1),
-    ([[], [], [('Ca',), ('Cb',)]], 2),
-    ([[('Aa',)], []], 1),
-    ([[('Aa',)], []], 1),
-    ([[('Aa',)], [], []], 1),
-    ([[('Aa',)], [], [('Ca',)]], 2),
-])
-def test_count(tree, expected):
-    model = _create_model(tree)
-    filter_model = sortfilter.CompletionFilterModel(model)
-    assert filter_model.count() == expected
-
-
-@pytest.mark.parametrize('pattern, dumb_sort, filter_cols, before, after', [
-    ('foo', None, [0],
-     [[('foo', '', ''), ('bar', '', '')]],
-     [[('foo', '', '')]]),
-
-    ('foo', None, [0],
-     [[('foob', '', ''), ('fooc', '', ''), ('fooa', '', '')]],
-     [[('fooa', '', ''), ('foob', '', ''), ('fooc', '', '')]]),
-
-    ('foo', None, [0],
-     [[('foo', '', '')], [('bar', '', '')]],
-     [[('foo', '', '')], []]),
-
-    # prefer foobar as it starts with the pattern
-    ('foo', None, [0],
-     [[('barfoo', '', ''), ('foobar', '', '')]],
-     [[('foobar', '', ''), ('barfoo', '', '')]]),
-
-    # however, don't rearrange categories
-    ('foo', None, [0],
-     [[('barfoo', '', '')], [('foobar', '', '')]],
-     [[('barfoo', '', '')], [('foobar', '', '')]]),
-
-    ('foo', None, [1],
-     [[('foo', 'bar', ''), ('bar', 'foo', '')]],
-     [[('bar', 'foo', '')]]),
-
-    ('foo', None, [0, 1],
-     [[('foo', 'bar', ''), ('bar', 'foo', ''), ('bar', 'bar', '')]],
-     [[('foo', 'bar', ''), ('bar', 'foo', '')]]),
-
-    ('foo', None, [0, 1, 2],
-     [[('foo', '', ''), ('bar', '')]],
-     [[('foo', '', '')]]),
-
-    # the fourth column is the sort role, which overrides data-based sorting
-    ('', None, [0],
-     [[('two', '', '', 2), ('one', '', '', 1), ('three', '', '', 3)]],
-     [[('one', '', ''), ('two', '', ''), ('three', '', '')]]),
-
-    ('', Qt.AscendingOrder, [0],
-     [[('two', '', '', 2), ('one', '', '', 1), ('three', '', '', 3)]],
-     [[('one', '', ''), ('two', '', ''), ('three', '', '')]]),
-
-    ('', Qt.DescendingOrder, [0],
-     [[('two', '', '', 2), ('one', '', '', 1), ('three', '', '', 3)]],
-     [[('three', '', ''), ('two', '', ''), ('one', '', '')]]),
-])
-def test_set_pattern(pattern, dumb_sort, filter_cols, before, after):
-    """Validate the filtering and sorting results of set_pattern."""
-    model = _create_model(before)
-    model.DUMB_SORT = dumb_sort
-    model.columns_to_filter = filter_cols
-    filter_model = sortfilter.CompletionFilterModel(model)
-    filter_model.set_pattern(pattern)
-    actual = _extract_model_data(filter_model)
-    assert actual == after
-
-
-def test_sort():
-    """Ensure that a sort argument passed to sort overrides DUMB_SORT.
-
-    While test_set_pattern above covers most of the sorting logic, this
-    particular case is easier to test separately.
-    """
-    model = _create_model([[('B', '', '', 1),
-                            ('C', '', '', 2),
-                            ('A', '', '', 0)]])
-    filter_model = sortfilter.CompletionFilterModel(model)
-
-    filter_model.sort(0, Qt.AscendingOrder)
-    actual = _extract_model_data(filter_model)
-    assert actual == [[('A', '', ''), ('B', '', ''), ('C', '', '')]]
-
-    filter_model.sort(0, Qt.DescendingOrder)
-    actual = _extract_model_data(filter_model)
-    assert actual == [[('C', '', ''), ('B', '', ''), ('A', '', '')]]

tests/unit/misc/test_lineparser.py

@@ -58,8 +58,8 @@ class TestBaseLineParser:
         mocker.patch('builtins.open', mock.mock_open())
 
         with lineparser._open('r'):
-            with pytest.raises(IOError, match="Refusing to double-open "
+            with pytest.raises(IOError,
-                                              "AppendLineParser."):
+                               match="Refusing to double-open LineParser."):
                 with lineparser._open('r'):
                     pass
 
@@ -115,7 +115,8 @@ class TestLineParser:
     def test_double_open(self, lineparser):
         """Test if save() bails on an already open file."""
         with lineparser._open('r'):
-            with pytest.raises(IOError):
+            with pytest.raises(IOError,
+                               match="Refusing to double-open LineParser."):
                 lineparser.save()
 
     def test_prepare_save(self, tmpdir, lineparser):
@@ -125,83 +126,3 @@ class TestLineParser:
         lineparser._prepare_save = lambda: False
         lineparser.save()
         assert (tmpdir / 'file').read() == 'pristine\n'
-
-
-class TestAppendLineParser:
-
-    BASE_DATA = ['old data 1', 'old data 2']
-
-    @pytest.fixture
-    def lineparser(self, tmpdir):
-        """Fixture to get an AppendLineParser for tests."""
-        lp = lineparsermod.AppendLineParser(str(tmpdir), 'file')
-        lp.new_data = self.BASE_DATA
-        lp.save()
-        return lp
-
-    def _get_expected(self, new_data):
-        """Get the expected data with newlines."""
-        return '\n'.join(self.BASE_DATA + new_data) + '\n'
-
-    def test_save(self, tmpdir, lineparser):
-        """Test save()."""
-        new_data = ['new data 1', 'new data 2']
-        lineparser.new_data = new_data
-        lineparser.save()
-        assert (tmpdir / 'file').read() == self._get_expected(new_data)
-
-    def test_clear(self, tmpdir, lineparser):
-        """Check if calling clear() empties both pending and persisted data."""
-        lineparser.new_data = ['one', 'two']
-        lineparser.save()
-        assert (tmpdir / 'file').read() == "old data 1\nold data 2\none\ntwo\n"
-
-        lineparser.new_data = ['one', 'two']
-        lineparser.clear()
-        lineparser.save()
-        assert not lineparser.new_data
-        assert (tmpdir / 'file').read() == ""
-
-    def test_iter_without_open(self, lineparser):
-        """Test __iter__ without having called open()."""
-        with pytest.raises(ValueError):
-            iter(lineparser)
-
-    def test_iter(self, lineparser):
-        """Test __iter__."""
-        new_data = ['new data 1', 'new data 2']
-        lineparser.new_data = new_data
-        with lineparser.open():
-            assert list(lineparser) == self.BASE_DATA + new_data
-
-    def test_iter_not_found(self, mocker):
-        """Test __iter__ with no file."""
-        open_mock = mocker.patch(
-            'qutebrowser.misc.lineparser.AppendLineParser._open')
-        open_mock.side_effect = FileNotFoundError
-        new_data = ['new data 1', 'new data 2']
-        linep = lineparsermod.AppendLineParser('foo', 'bar')
-        linep.new_data = new_data
-        with linep.open():
-            assert list(linep) == new_data
-
-    def test_get_recent_none(self, tmpdir):
-        """Test get_recent with no data."""
-        (tmpdir / 'file2').ensure()
-        linep = lineparsermod.AppendLineParser(str(tmpdir), 'file2')
-        assert linep.get_recent() == []
-
-    def test_get_recent_little(self, lineparser):
-        """Test get_recent with little data."""
-        data = [e + '\n' for e in self.BASE_DATA]
-        assert lineparser.get_recent() == data
-
-    def test_get_recent_much(self, lineparser):
-        """Test get_recent with much data."""
-        size = 64
-        new_data = ['new data {}'.format(i) for i in range(size)]
-        lineparser.new_data = new_data
-        lineparser.save()
-        data = os.linesep.join(self.BASE_DATA + new_data) + os.linesep
-        data = [e + '\n' for e in data[-size:].splitlines()]
-        assert lineparser.get_recent(size) == data

tests/unit/misc/test_sql.py

@@ -0,0 +1,179 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016-2017 Ryan Roden-Corrent (rcorre) <ryan@rcorre.net>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the SQL API."""
+
+import pytest
+from qutebrowser.misc import sql
+
+
+pytestmark = pytest.mark.usefixtures('init_sql')
+
+
+def test_init():
+    sql.SqlTable('Foo', ['name', 'val', 'lucky'])
+    # should not error if table already exists
+    sql.SqlTable('Foo', ['name', 'val', 'lucky'])
+
+
+def test_insert(qtbot):
+    table = sql.SqlTable('Foo', ['name', 'val', 'lucky'])
+    with qtbot.waitSignal(table.changed):
+        table.insert({'name': 'one', 'val': 1, 'lucky': False})
+    with qtbot.waitSignal(table.changed):
+        table.insert({'name': 'wan', 'val': 1, 'lucky': False})
+
+
+def test_insert_replace(qtbot):
+    table = sql.SqlTable('Foo', ['name', 'val', 'lucky'],
+                         constraints={'name': 'PRIMARY KEY'})
+    with qtbot.waitSignal(table.changed):
+        table.insert({'name': 'one', 'val': 1, 'lucky': False}, replace=True)
+    with qtbot.waitSignal(table.changed):
+        table.insert({'name': 'one', 'val': 11, 'lucky': True}, replace=True)
+    assert list(table) == [('one', 11, True)]
+
+    with pytest.raises(sql.SqlException):
+        table.insert({'name': 'one', 'val': 11, 'lucky': True}, replace=False)
+
+
+def test_insert_batch(qtbot):
+    table = sql.SqlTable('Foo', ['name', 'val', 'lucky'])
+
+    with qtbot.waitSignal(table.changed):
+        table.insert_batch({'name': ['one', 'nine', 'thirteen'],
+                            'val': [1, 9, 13],
+                            'lucky': [False, False, True]})
+
+    assert list(table) == [('one', 1, False),
+                           ('nine', 9, False),
+                           ('thirteen', 13, True)]
+
+
+def test_insert_batch_replace(qtbot):
+    table = sql.SqlTable('Foo', ['name', 'val', 'lucky'],
+                         constraints={'name': 'PRIMARY KEY'})
+
+    with qtbot.waitSignal(table.changed):
+        table.insert_batch({'name': ['one', 'nine', 'thirteen'],
+                            'val': [1, 9, 13],
+                            'lucky': [False, False, True]})
+
+    with qtbot.waitSignal(table.changed):
+        table.insert_batch({'name': ['one', 'nine'],
+                            'val': [11, 19],
+                            'lucky': [True, True]},
+                           replace=True)
+
+    assert list(table) == [('thirteen', 13, True),
+                           ('one', 11, True),
+                           ('nine', 19, True)]
+
+    with pytest.raises(sql.SqlException):
+        table.insert_batch({'name': ['one', 'nine'],
+                            'val': [11, 19],
+                            'lucky': [True, True]})
+
+
+def test_iter():
+    table = sql.SqlTable('Foo', ['name', 'val', 'lucky'])
+    table.insert({'name': 'one', 'val': 1, 'lucky': False})
+    table.insert({'name': 'nine', 'val': 9, 'lucky': False})
+    table.insert({'name': 'thirteen', 'val': 13, 'lucky': True})
+    assert list(table) == [('one', 1, False),
+                           ('nine', 9, False),
+                           ('thirteen', 13, True)]
+
+
+@pytest.mark.parametrize('rows, sort_by, sort_order, limit, result', [
+    ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'a', 'asc', 5,
+        [(1, 6), (2, 5), (3, 4)]),
+    ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'a', 'desc', 3,
+        [(3, 4), (2, 5), (1, 6)]),
+    ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'b', 'desc', 2,
+        [(1, 6), (2, 5)]),
+    ([{"a": 2, "b": 5}, {"a": 1, "b": 6}, {"a": 3, "b": 4}], 'a', 'asc', -1,
+        [(1, 6), (2, 5), (3, 4)]),
+])
+def test_select(rows, sort_by, sort_order, limit, result):
+    table = sql.SqlTable('Foo', ['a', 'b'])
+    for row in rows:
+        table.insert(row)
+    assert list(table.select(sort_by, sort_order, limit)) == result
+
+
+def test_delete(qtbot):
+    table = sql.SqlTable('Foo', ['name', 'val', 'lucky'])
+    table.insert({'name': 'one', 'val': 1, 'lucky': False})
+    table.insert({'name': 'nine', 'val': 9, 'lucky': False})
+    table.insert({'name': 'thirteen', 'val': 13, 'lucky': True})
+    with pytest.raises(KeyError):
+        table.delete('name', 'nope')
+    with qtbot.waitSignal(table.changed):
+        table.delete('name', 'thirteen')
+    assert list(table) == [('one', 1, False), ('nine', 9, False)]
+    with qtbot.waitSignal(table.changed):
+        table.delete('lucky', False)
+    assert not list(table)
+
+
+def test_len():
+    table = sql.SqlTable('Foo', ['name', 'val', 'lucky'])
+    assert len(table) == 0
+    table.insert({'name': 'one', 'val': 1, 'lucky': False})
+    assert len(table) == 1
+    table.insert({'name': 'nine', 'val': 9, 'lucky': False})
+    assert len(table) == 2
+    table.insert({'name': 'thirteen', 'val': 13, 'lucky': True})
+    assert len(table) == 3
+
+
+def test_contains():
+    table = sql.SqlTable('Foo', ['name', 'val', 'lucky'])
+    table.insert({'name': 'one', 'val': 1, 'lucky': False})
+    table.insert({'name': 'nine', 'val': 9, 'lucky': False})
+    table.insert({'name': 'thirteen', 'val': 13, 'lucky': True})
+
+    name_query = table.contains_query('name')
+    val_query = table.contains_query('val')
+    lucky_query = table.contains_query('lucky')
+
+    assert name_query.run(val='one').value()
+    assert name_query.run(val='thirteen').value()
+    assert val_query.run(val=9).value()
+    assert lucky_query.run(val=False).value()
+    assert lucky_query.run(val=True).value()
+    assert not name_query.run(val='oone').value()
+    assert not name_query.run(val=1).value()
+    assert not name_query.run(val='*').value()
+    assert not val_query.run(val=10).value()
+
+
+def test_delete_all(qtbot):
+    table = sql.SqlTable('Foo', ['name', 'val', 'lucky'])
+    table.insert({'name': 'one', 'val': 1, 'lucky': False})
+    table.insert({'name': 'nine', 'val': 9, 'lucky': False})
+    table.insert({'name': 'thirteen', 'val': 13, 'lucky': True})
+    with qtbot.waitSignal(table.changed):
+        table.delete_all()
+    assert list(table) == []
+
+
+def test_version():
+    assert isinstance(sql.version(), str)

tests/unit/utils/test_utils.py

@@ -759,37 +759,6 @@ def test_sanitize_filename_empty_replacement():
     assert utils.sanitize_filename(name, replacement=None) == 'Bad File'
 
 
-class TestNewestSlice:
-
-    """Test newest_slice."""
-
-    def test_count_minus_two(self):
-        """Test with a count of -2."""
-        with pytest.raises(ValueError):
-            utils.newest_slice([], -2)
-
-    @pytest.mark.parametrize('items, count, expected', [
-        # Count of -1 (all elements).
-        (range(20), -1, range(20)),
-        # Count of 0 (no elements).
-        (range(20), 0, []),
-        # Count which is much smaller than the iterable.
-        (range(20), 5, [15, 16, 17, 18, 19]),
-        # Count which is exactly one smaller."""
-        (range(5), 4, [1, 2, 3, 4]),
-        # Count which is just as large as the iterable."""
-        (range(5), 5, range(5)),
-        # Count which is one bigger than the iterable.
-        (range(5), 6, range(5)),
-        # Count which is much bigger than the iterable.
-        (range(5), 50, range(5)),
-    ])
-    def test_good(self, items, count, expected):
-        """Test slices which shouldn't raise an exception."""
-        sliced = utils.newest_slice(items, count)
-        assert list(sliced) == list(expected)
-
-
 class TestGetSetClipboard:
 
     @pytest.fixture(autouse=True)

tests/unit/utils/test_version.py

@@ -849,6 +849,7 @@ def test_version_output(params, stubs, monkeypatch):
                          if params.style else
                          stubs.FakeQApplication(instance=None)),
         'QLibraryInfo.location': (lambda _loc: 'QT PATH'),
+        'sql.version': lambda: 'SQLITE VERSION',
     }
 
     substitutions = {
@@ -909,6 +910,7 @@ def test_version_output(params, stubs, monkeypatch):
         MODULE VERSION 1
         MODULE VERSION 2
         pdf.js: PDFJS VERSION
+        sqlite: SQLITE VERSION
         QtNetwork SSL: {ssl}
         {style}
         Platform: PLATFORM, ARCHITECTURE{linuxdist}