Merge branch 'master' of https://github.com/qutebrowser/qutebrowser into jay/insert-no-leave

.coveragerc

@@ -14,6 +14,7 @@ exclude_lines =
     raise NotImplementedError
     raise utils\.Unreachable
     if __name__ == ["']__main__["']:
+    if MYPY:
 
 [xml]
 output=coverage.xml

.flake8

@@ -33,6 +33,7 @@ exclude = .*,__pycache__,resources.py
 #       (false-positives)
 # D413: Missing blank line after last section (not in pep257?)
 # A003: Builtin name for class attribute (needed for overridden methods)
+# W504: line break after binary operator
 ignore =
     B001,B008,B305,
     E128,E226,E265,E501,E402,E266,E722,E731,
@@ -40,10 +41,12 @@ ignore =
     N802,
     P101,P102,P103,
     D102,D103,D106,D107,D104,D105,D209,D211,D401,D402,D403,D413,
-    A003
+    A003,
+    W504
 min-version = 3.4.0
 max-complexity = 12
 per-file-ignores =
+    /qutebrowser/api/hook.py : N801
     /tests/**/*.py : D100,D101,D401
     /tests/unit/browser/test_history.py : N806
     /tests/helpers/fixtures.py : N806

.gitignore

@@ -41,3 +41,4 @@ TODO
 /scripts/testbrowser/cpp/webengine/.qmake.stash
 /scripts/dev/pylint_checkers/qute_pylint.egg-info
 /misc/file_version_info.txt
+/doc/extapi/_build

.travis.yml

@@ -1,37 +1,26 @@
-sudo: false
+dist: xenial
-dist: trusty
 language: python
 group: edge
 python: 3.6
+os: linux
 
 matrix:
   include:
-    - os: linux
+    - env: DOCKER=archlinux
-      env: DOCKER=archlinux
       services: docker
-    - os: linux
+    - env: DOCKER=archlinux-webengine QUTE_BDD_WEBENGINE=true
-      env: DOCKER=archlinux-webengine QUTE_BDD_WEBENGINE=true
       services: docker
-    - os: linux
+    - env: TESTENV=py36-pyqt571
-      env: TESTENV=py36-pyqt571
+    - python: 3.5
-    - os: linux
-      python: 3.5
       env: TESTENV=py35-pyqt571
-    - os: linux
+    - env: TESTENV=py36-pyqt59
-      env: TESTENV=py36-pyqt59
+    - env: TESTENV=py36-pyqt510
-    - os: linux
-      env: TESTENV=py36-pyqt510
       addons:
         apt:
           packages:
           - xfonts-base
-    - os: linux
+    - env: TESTENV=py36-pyqt511-cov
-      env: TESTENV=py36-pyqt511-cov
+    - python: 3.7
-    # https://github.com/travis-ci/travis-ci/issues/9069
-    - os: linux
-      python: 3.7
-      sudo: required
-      dist: xenial
       env: TESTENV=py37-pyqt511
     - os: osx
       env: TESTENV=py37 OSX=sierra
@@ -41,38 +30,26 @@ matrix:
     # - os: osx
     #   env: TESTENV=py35 OSX=yosemite
     #   osx_image: xcode6.4
-    - os: linux
+    - env: TESTENV=pylint
-      env: TESTENV=pylint PYTHON=python3.6
+    - env: TESTENV=flake8
-    - os: linux
+    - env: TESTENV=mypy
-      env: TESTENV=flake8
+    - env: TESTENV=docs
-    - os: linux
-      env: TESTENV=docs
       addons:
          apt:
            packages:
            - asciidoc
-    - os: linux
+    - env: TESTENV=vulture
-      env: TESTENV=vulture
+    - env: TESTENV=misc
-    - os: linux
+    - env: TESTENV=pyroma
-      env: TESTENV=misc
+    - env: TESTENV=check-manifest
-    - os: linux
+    - env: TESTENV=eslint
-      env: TESTENV=pyroma
-    - os: linux
-      env: TESTENV=check-manifest
-    - os: linux
-      env: TESTENV=eslint
       language: node_js
       python: null
       node_js: "lts/*"
-    - os: linux
+    - language: generic
-      language: generic
       env: TESTENV=shellcheck
       services: docker
   fast_finish: true
-  allow_failures:
-    # https://github.com/qutebrowser/qutebrowser/issues/4055
-    - os: linux
-      env: TESTENV=py36-pyqt510
 
 cache:
   directories:
@@ -91,16 +68,3 @@ after_success:
 
 after_failure:
   - bash scripts/dev/ci/travis_backtrace.sh
-
-notifications:
-  webhooks:
-    - https://buildtimetrend.herokuapp.com/travis
-  irc:
-    channels:
-      - "chat.freenode.net#qutebrowser-dev"
-    on_success: always
-    on_failure: always
-    skip_join: true
-    template:
-      - "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}"
-      - "%{compare_url} - %{build_url}"

MANIFEST.in

@@ -13,7 +13,7 @@ include qutebrowser/utils/testfile
 include qutebrowser/git-commit-id
 include LICENSE doc/* README.asciidoc
 include misc/qutebrowser.desktop
-include misc/qutebrowser.appdata.xml
+include misc/org.qutebrowser.qutebrowser.appdata.xml
 include misc/Makefile
 include requirements.txt
 include tox.ini
@@ -32,6 +32,7 @@ include doc/changelog.asciidoc
 prune tests
 prune qutebrowser/3rdparty
 exclude pytest.ini
+exclude mypy.ini
 exclude qutebrowser/javascript/.eslintrc.yaml
 exclude qutebrowser/javascript/.eslintignore
 exclude doc/help
@@ -39,5 +40,6 @@ exclude .*
 exclude misc/qutebrowser.spec
 exclude misc/qutebrowser.nsi
 exclude misc/qutebrowser.rcc
+prune doc/extapi
 
 global-exclude __pycache__ *.pyc *.pyo

README.asciidoc

@@ -154,7 +154,11 @@ https://www.macstadium.com/opensource[Open Source Project].
 (They don't require including this here - I've just been very happy with their
 offer, and without them, no macOS releases or tests would exist)
 
+Thanks to the https://www.hsr.ch/[HSR Hochschule für Technik Rapperswil], which
+made it possible to work on qutebrowser extensions as a student research project.
+
 image:.github/img/macstadium.png["powered by MacStadium",width=200,link="https://www.macstadium.com/"]
+image:.github/img/hsr.png["HSR Hochschule für Technik Rapperswil",link="https://www.hsr.ch/"]
 
 Authors
 -------

doc/changelog.asciidoc

@@ -25,6 +25,7 @@ Added
   opened from a page should stack on each other or not.
 - New `completion.open_categories` setting which allows to configure which
   categories are shown in the `:open` completion, and how they are ordered.
+- New `tabs.pinned.frozen` setting to allow/deny navigating in pinned tabs.
 - New config manipulation commands:
   * `:config-dict-add` and `:config-list-add` to a new element to a dict/list
     setting.
@@ -32,6 +33,8 @@ Added
     dict/list setting.
 - New `hints.selectors` setting which allows to configure what CSS selectors
   are used for hints, and also allows adding custom hint groups.
+- New `:yank markdown` feature which yanks the current URL and title in
+  markdown format.
 
 Changed
 ~~~~~~~
@@ -40,7 +43,7 @@ Changed
   (`:close`), while `:qa` quits (`:quit`). The behavior of `:wq` remains
   unchanged (`:quit --save`), as closing a window while saving the session
   doesn't make sense.
-- Completion highlighting is now done differently (using QSyntaxHighlither),
+- Completion highlighting is now done differently (using `QSyntaxHighlighter`),
   which should fix some highlighting corner-cases.
 - The `QtColor` config type now also understands colors like `rgb(...)`.
 - `:yank` now has a `--quiet` option which causes it to not display a message.
@@ -48,6 +51,14 @@ Changed
 - The `content.host_blocking.enabled` setting now supports URL patterns, so the
   adblocker can be disabled on a given page.
 - Elements with a `tabindex` attribute now also get hints by default.
+- Various small performance improvements for hints and the completion.
+- The Wayland check for QtWebEngine is now disabled on Qt >= 5.11.2, as those
+  versions should work without any issues.
+- The JavaScript `console` object is now available in PAC files.
+- The metainfo file `qutebrowser.appdata.xml` is now renamed to
+  `org.qutebrowser.qutebrowser.appdata.xml`.
+- The `qute-pass` userscript now understands domains in gpg filenames
+  in addition to directory names.
 
 Fixed
 ~~~~~
@@ -55,6 +66,29 @@ Fixed
 - Invalid world IDs now get rejected for `:jseval` and GreaseMonkey scripts.
 - When websites suggest download filenames with invalid characters, those are
   now correctly replaced.
+- Invalid hint length calculation in certain rare cases.
+- Dragging tabs in the tab bar (which was broken in v1.5.0)
+- Using Shift-Home in command mode now works properly.
+- Workaround for a Qt bug which prevented
+  `content.cookies.accept = no-3rdparty` from working properly on some pages
+  like GMail. However, the default for `content.cookies.accept` is still `all`
+  to be in line with what other browsers do.
+- `:navigate` not incrementing in anchors or queries.
+- Crash when trying to use a proxy requiring authentication with QtWebKit.
+- Slashes in search terms are now percent-escaped.
+- When `scrolling.bar = True` was set in versions before v1.5.0, this now
+  correctly gets migrated to `always` instead of `when-searching`.
+- Completion highlighting now works again on Qt 5.11.3 and 5.12.1.
+
+v1.5.2
+------
+
+Changed
+~~~~~~~
+
+- The `content.cookies.accept` setting is now set to `all` instead of
+  `no-3rdparty` by default, as `no-3rdparty` breaks various pages such as
+  GMail.
 
 v1.5.1
 ------
@@ -1223,7 +1257,7 @@ Added
 - New `:debug-log-filter` command to change console log filtering on-the-fly.
 - New `:debug-log-level` command to change the console loglevel on-the-fly.
 - New `general -> yank-ignored-url-parameters` option to configure which URL
-  parameters (like `utm_source` etc.) to strip off when yanking an URL.
+  parameters (like `utm_source` etc.) to strip off when yanking a URL.
 - Support for the
   https://developer.mozilla.org/en-US/docs/Web/API/Page_Visibility_API[HTML5 page visibility API]
 - New `readability` userscript which shows a readable version of a page (using
@@ -1334,7 +1368,7 @@ Changed
 - `:hint` has a new `--add-history` argument to add the URL to the history for
   yank/spawn targets.
 - `:set` now cycles through values if more than one argument is given.
-- `:open` now opens `default-page` without an URL even without `-t`/`-b`/`-w` given.
+- `:open` now opens `default-page` without a URL even without `-t`/`-b`/`-w` given.
 
 Deprecated
 ~~~~~~~~~~

doc/contributing.asciidoc

@@ -407,7 +407,7 @@ Creating a new command is straightforward:
 
 [source,python]
 ----
-import qutebrowser.commands.cmdutils
+from qutebrowser.api import cmdutils
 
 ...
 
@@ -429,7 +429,7 @@ selects which object registry (global, per-tab, etc.) to use.  See the
 
 There are also other arguments to customize the way the command is
 registered; see the class documentation for `register` in
-`qutebrowser.commands.cmdutils` for details.
+`qutebrowser.api.cmdutils` for details.
 
 The types of the function arguments are inferred based on their default values,
 e.g., an argument `foo=True` will be converted to a flag `-f`/`--foo` in
@@ -480,8 +480,10 @@ For `typing.Union` types, the given `choices` are only checked if other types
 The following arguments are supported for `@cmdutils.argument`:
 
 - `flag`: Customize the short flag (`-x`) the argument will get.
-- `win_id=True`: Mark the argument as special window ID argument.
+- `value`: Tell qutebrowser to fill the argument with special values:
-- `count=True`: Mark the argument as special count argument.
+   - `value=cmdutils.Value.count`: The `count` given by the user to the command.
+   - `value=cmdutils.Value.win_id`: The window ID of the current window.
+   - `value=cmdutils.Value.cur_tab`: The tab object which is currently focused.
 - `completion`: A completion function (see `qutebrowser.completions.models.*`)
   to use when completing arguments for the given command.
 - `choices`: The allowed string choices for the argument.

doc/extapi/api.rst

@@ -0,0 +1,48 @@
+API modules
+===========
+
+cmdutils module
+---------------
+
+.. automodule:: qutebrowser.api.cmdutils
+   :members:
+   :imported-members:
+
+apitypes module
+---------------
+
+.. automodule:: qutebrowser.api.apitypes
+   :members:
+   :imported-members:
+
+config module
+-------------
+
+.. automodule:: qutebrowser.api.config
+   :members:
+
+downloads module
+----------------
+
+.. automodule:: qutebrowser.api.downloads
+   :members:
+
+hook module
+-----------
+
+.. automodule:: qutebrowser.api.hook
+   :members:
+
+interceptor module
+------------------
+
+.. automodule:: qutebrowser.api.interceptor
+   :members:
+   :imported-members:
+
+message module
+--------------
+
+.. automodule:: qutebrowser.api.message
+   :members:
+   :imported-members:

doc/extapi/conf.py

@@ -0,0 +1,179 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'qutebrowser extensions'
+copyright = '2018, Florian Bruhin'
+author = 'Florian Bruhin'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = ''
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.napoleon',
+]
+autodoc_member_order = 'bysource'
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'qutebrowserextensionsdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'qutebrowserextensions.tex', 'qutebrowser extensions Documentation',
+     'Florian Bruhin', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'qutebrowserextensions', 'qutebrowser extensions Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'qutebrowserextensions', 'qutebrowser extensions Documentation',
+     author, 'qutebrowserextensions', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------

doc/extapi/index.rst

@@ -0,0 +1,22 @@
+.. qutebrowser extensions documentation master file, created by
+   sphinx-quickstart on Tue Dec 11 18:59:44 2018.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to qutebrowser extensions's documentation!
+==================================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   api
+   tab
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

doc/extapi/tab.rst

@@ -0,0 +1,44 @@
+Tab API
+=======
+
+.. autoclass:: qutebrowser.browser.browsertab.AbstractTab()
+   :members:
+
+.. autoclass:: qutebrowser.browser.browsertab.AbstractAction()
+   :members:
+
+.. autoclass:: qutebrowser.browser.browsertab.AbstractPrinting()
+   :members:
+
+.. autoclass:: qutebrowser.browser.browsertab.AbstractSearch()
+   :members:
+
+.. autoclass:: qutebrowser.browser.browsertab.AbstractZoom()
+   :members:
+
+.. autoclass:: qutebrowser.browser.browsertab.AbstractCaret()
+   :members:
+
+.. autoclass:: qutebrowser.browser.browsertab.AbstractScroller()
+   :members:
+
+.. autoclass:: qutebrowser.browser.browsertab.AbstractHistory()
+   :members:
+
+.. autoclass:: qutebrowser.browser.browsertab.AbstractElements()
+   :members:
+
+.. autoclass:: qutebrowser.browser.browsertab.AbstractAudio()
+   :members:
+
+Web element API
+===============
+
+.. autoclass:: qutebrowser.browser.webelem.AbstractWebElement
+   :members:
+
+.. autoclass:: qutebrowser.browser.webelem.Error
+   :members:
+
+.. autoclass:: qutebrowser.browser.webelem.OrphanedError
+   :members:

doc/faq.asciidoc

@@ -211,15 +211,16 @@ Why does J move to the next (right) tab, and K to the previous (left) one?::
 
 What's the difference between insert and passthrough mode?::
     They are quite similar, but insert mode has some bindings (like `Ctrl-e` to
-    open an editor) while passthrough mode only has escape bound. It might also
+    open an editor) while passthrough mode only has shift+escape bound. This is
-    be useful to rebind escape to something else in passthrough mode only, to be
+    because shift+escape is unlikely to be a useful binding to be passed to a
-    able to send an escape keypress to the website.
+    webpage. However, any other keys may be assigned to leaving passthrough mode
+    instead of shift+escape should this be desired.
 
-Why takes it longer to open an URL in qutebrowser than in chromium?::
+Why does it take longer to open a URL in qutebrowser than in chromium?::
-    When opening an URL in an existing instance the normal qutebrowser
+    When opening a URL in an existing instance, the normal qutebrowser
     Python script is started and a few PyQt libraries need to be
     loaded until it is detected that there is an instance running
-    where the URL is then passed to. This takes some time.
+    to which the URL is then passed. This takes some time.
     One workaround is to use this
     https://github.com/qutebrowser/qutebrowser/blob/master/scripts/open_url_in_instance.sh[script]
     and place it in your $PATH with the name "qutebrowser". This
@@ -260,6 +261,12 @@ Note that there are some missing features which you may run into:
 . Any greasemonkey API function to do with adding UI elements is not currently
   supported. That means context menu extentensions and background pages.
 
+How do I change the `WM_CLASS` used by qutebrowser windows?::
+    Qt only supports setting `WM_CLASS` globally, which you can do by starting
+    with `--qt-arg name foo`. Note that all windows are part of the same
+    qutebrowser instance (unless you use `--temp-basedir` or `--basedir`), so
+    they all will share the same `WM_CLASS`.
+
 == Troubleshooting
 
 Unable to view flash content.::

doc/help/commands.asciidoc

@@ -1472,6 +1472,7 @@ Yank something to the clipboard or primary selection.
  - `title`: The current page's title.
  - `domain`: The current scheme, domain, and port number.
  - `selection`: The selection under the cursor.
+ - `markdown`: Yank title and URL in markdown format.
  
 
 
@@ -1483,14 +1484,14 @@ Yank something to the clipboard or primary selection.
 
 [[zoom]]
 === zoom
-Syntax: +:zoom [*--quiet*] ['zoom']+
+Syntax: +:zoom [*--quiet*] ['level']+
 
 Set the zoom level for the current tab.
 
 The zoom can be given as argument or as [count]. If neither is given, the zoom is set to the default zoom. If both are given, use [count].
 
 ==== positional arguments
-* +'zoom'+: The zoom percentage to set.
+* +'level'+: The zoom percentage to set.
 
 ==== optional arguments
 * +*-q*+, +*--quiet*+: Don't show a zoom level message.

doc/help/configuring.asciidoc

@@ -19,10 +19,10 @@ hand, you can simply use those - see
 <<autoconfig,"Configuring qutebrowser via the user interface">> for details.
 
 For more advanced configuration, you can write a `config.py` file - see
-<<configpy,"Configuring qutebrowser via config.py">>. As soon as a `config.py`
+<<configpy,"Configuring qutebrowser via config.py">>. When a `config.py`
 exists, the `autoconfig.yml` file **is not read anymore** by default. You need
-to <<configpy-autoconfig,load it by hand>> if you want settings done via
+to <<configpy-autoconfig,load it from `config.py`>> if you want settings changed via
-`:set`/`:bind` to still persist.
+`:set`/`:bind` to persist between restarts.
 
 [[autoconfig]]
 Configuring qutebrowser via the user interface
@@ -229,18 +229,18 @@ Loading `autoconfig.yml`
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 All customization done via the UI (`:set`, `:bind` and `:unbind`) is
-stored in the `autoconfig.yml` file, which is not loaded automatically as soon
+stored in the `autoconfig.yml` file. When a `config.py` file exists, `autoconfig.yml`
-as a `config.py` exists. If you want those settings to be loaded, you'll need to
+is not loaded automatically. To load `autoconfig.yml` automatically, add the
-explicitly load the `autoconfig.yml` file in your `config.py` by doing:
+following snippet to `config.py`:
 
-.config.py:
 [source,python]
 ----
 config.load_autoconfig()
 ----
 
-If you do so at the top of your file, your `config.py` settings will take
+You can configure which file overrides the other by the location of the above code snippet.
-precedence as they overwrite the settings done in `autoconfig.yml`.
+Place the snippet at the top to allow `config.py` to override `autoconfig.yml`.
+Place the snippet at the bottom for the opposite effect.
 
 Importing other modules
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -396,6 +396,7 @@ Pre-built colorschemes
 - A collection of https://github.com/chriskempson/base16[base16] color-schemes can be found in https://github.com/theova/base16-qutebrowser[base16-qutebrowser] and used with https://github.com/AuditeMarlow/base16-manager[base16-manager].
 - Two implementations of the https://github.com/arcticicestudio/nord[Nord] colorscheme for qutebrowser exist: https://github.com/Linuus/nord-qutebrowser[Linuus], https://github.com/KnownAsDon/QuteBrowser-Nord-Theme[KnownAsDon]
 - https://github.com/evannagle/qutebrowser-dracula-theme[Dracula]
+- https://github.com/jjzmajic/qutewal[Pywal theme]
 
 Avoiding flake8 errors
 ^^^^^^^^^^^^^^^^^^^^^^
@@ -423,6 +424,19 @@ config = config  # type: ConfigAPI # noqa: F821 pylint: disable=E0602,C0103
 c = c  # type: ConfigContainer # noqa: F821 pylint: disable=E0602,C0103
 ----
 
+emacs-like config
+^^^^^^^^^^^^^^^^^
+
+Various emacs/conkeror-like keybinding configs exist:
+
+- https://gitlab.com/jgkamat/qutemacs/blob/master/qutemacs.py[jgkamat]
+- https://gitlab.com/Kaligule/qutebrowser-emacs-config/blob/master/config.py[Kaligule]
+- http://me0w.net/pit/1540882719[nm0i]
+
+It's also mostly possible to get rid of modal keybindings by setting
+`input.insert_mode.auto_enter` to `false`, and `input.forward_unbound_keys` to
+`all`.
+
 [[migrating]]
 Migrating older configurations
 ------------------------------

doc/help/settings.asciidoc

@@ -261,6 +261,7 @@
 |<<tabs.new_position.stacking,tabs.new_position.stacking>>|Stack related tabs on top of each other when opened consecutively.
 |<<tabs.new_position.unrelated,tabs.new_position.unrelated>>|Position of new tabs which are not opened from another tab.
 |<<tabs.padding,tabs.padding>>|Padding (in pixels) around text for tabs.
+|<<tabs.pinned.frozen,tabs.pinned.frozen>>|Force pinned tabs to stay at fixed URL.
 |<<tabs.pinned.shrink,tabs.pinned.shrink>>|Shrink pinned tabs down to their contents.
 |<<tabs.position,tabs.position>>|Position of the tab bar.
 |<<tabs.select_on_remove,tabs.select_on_remove>>|Which tab to select when the focused tab is removed.
@@ -640,10 +641,12 @@ Default:
 * +pass:[xO]+: +pass:[set-cmd-text :open -b -r {url:pretty}]+
 * +pass:[xo]+: +pass:[set-cmd-text -s :open -b]+
 * +pass:[yD]+: +pass:[yank domain -s]+
+* +pass:[yM]+: +pass:[yank markdown -s]+
 * +pass:[yP]+: +pass:[yank pretty-url -s]+
 * +pass:[yT]+: +pass:[yank title -s]+
 * +pass:[yY]+: +pass:[yank -s]+
 * +pass:[yd]+: +pass:[yank domain]+
+* +pass:[ym]+: +pass:[yank markdown]+
 * +pass:[yp]+: +pass:[yank pretty-url]+
 * +pass:[yt]+: +pass:[yank title]+
 * +pass:[yy]+: +pass:[yank]+
@@ -1591,11 +1594,11 @@ Type: <<types,String>>
 Valid values:
 
  * +all+: Accept all cookies.
- * +no-3rdparty+: Accept cookies from the same origin only.
+ * +no-3rdparty+: Accept cookies from the same origin only. This is known to break some sites, such as GMail.
  * +no-unknown-3rdparty+: Accept cookies from the same origin only, unless a cookie is already set for the domain. On QtWebEngine, this is the same as no-3rdparty.
  * +never+: Don't accept cookies at all.
 
-Default: +pass:[no-3rdparty]+
+Default: +pass:[all]+
 
 On QtWebEngine, this setting requires Qt 5.11 or newer.
 
@@ -2958,7 +2961,7 @@ Default: +pass:[false]+
 === search.ignore_case
 When to find text on a page case-insensitively.
 
-Type: <<types,String>>
+Type: <<types,IgnoreCase>>
 
 Valid values:
 
@@ -3305,6 +3308,14 @@ Default:
 - +pass:[right]+: +pass:[5]+
 - +pass:[top]+: +pass:[0]+
 
+[[tabs.pinned.frozen]]
+=== tabs.pinned.frozen
+Force pinned tabs to stay at fixed URL.
+
+Type: <<types,Bool>>
+
+Default: +pass:[true]+
+
 [[tabs.pinned.shrink]]
 === tabs.pinned.shrink
 Shrink pinned tabs down to their contents.
@@ -3622,6 +3633,7 @@ Lists with duplicate flags are invalid. Each item is checked against the valid v
 |FontFamily|A Qt font family.
 |FormatString|A string with placeholders.
 |FuzzyUrl|A URL which gets interpreted as search if needed.
+|IgnoreCase|Whether to search case insensitively.
 |Int|Base class for an integer setting.
 |Key|A name of a key.
 |List|A list of values.

doc/install.asciidoc

@@ -102,18 +102,12 @@ $ python3 scripts/asciidoc2html.py
 On Fedora
 ---------
 
-NOTE: Fedora's packages used to be outdated for a long time, but are
-now (November 2017) maintained and up-to-date again.
-
 qutebrowser is available in the official repositories:
 
 -----
 # dnf install qutebrowser
 -----
 
-However, note that Fedora 25/26 won't be updated to qutebrowser v1.0, so you
-might want to <<tox,install qutebrowser via tox>> instead there.
-
 Additional hints
 ~~~~~~~~~~~~~~~~
 

misc/Makefile

@@ -17,8 +17,8 @@ doc/qutebrowser.1.html:
 
 install: doc/qutebrowser.1.html
 	$(PYTHON) setup.py install --prefix="$(PREFIX)" --optimize=1 $(SETUPTOOLSOPTS)
-	install -Dm644 misc/qutebrowser.appdata.xml \
+	install -Dm644 misc/org.qutebrowser.qutebrowser.appdata.xml \
-		"$(DESTDIR)$(DATADIR)/metainfo/qutebrowser.appdata.xml"
+		"$(DESTDIR)$(DATADIR)/metainfo/org.qutebrowser.qutebrowser.appdata.xml"
 	install -Dm644 doc/qutebrowser.1 \
 		"$(DESTDIR)$(MANDIR)/man1/qutebrowser.1"
 	install -Dm644 misc/qutebrowser.desktop \

misc/qutebrowser.desktop

@@ -1,7 +1,45 @@
 [Desktop Entry]
 Name=qutebrowser
 GenericName=Web Browser
+GenericName[ar]=ﻢﺘﺼﻔﺣ ﺎﻠﺸﺒﻛﺓ
+GenericName[bg]=Уеб браузър
+GenericName[ca]=Navegador web
+GenericName[cs]=WWW prohlížeč
+GenericName[da]=Browser
+GenericName[de]=Web-Browser
+GenericName[el]=Περιηγητής ιστού
+GenericName[en_GB]=Web Browser
+GenericName[es]=Navegador web
+GenericName[et]=Veebibrauser
+GenericName[fi]=WWW-selain
+GenericName[fr]=Navigateur Web
+GenericName[gu]=વેબ બ્રાઉઝર
+GenericName[he]=דפדפן אינטרנט
+GenericName[hi]=वेब ब्राउज़र
+GenericName[hu]=Webböngésző
+GenericName[it]=Browser Web
+GenericName[ja]=ウェブブラウザ
+GenericName[kn]=ಜಾಲ ವೀಕ್ಷಕ
+GenericName[ko]=웹 브라우저
+GenericName[lt]=Žiniatinklio naršyklė
+GenericName[lv]=Tīmekļa pārlūks
+GenericName[ml]=വെബ് ബ്രൌസര്<200d>
+GenericName[mr]=वेब ब्राऊजर
+GenericName[nb]=Nettleser
+GenericName[nl]=Webbrowser
+GenericName[pl]=Przeglądarka WWW
+GenericName[pt]=Navegador Web
+GenericName[pt_BR]=Navegador da Internet
+GenericName[ro]=Navigator de Internet
+GenericName[ru]=Веб-браузер
+GenericName[sl]=Spletni brskalnik
+GenericName[sv]=Webbläsare
+GenericName[ta]=இணைய உலாவி
+GenericName[th]=เว็บเบราว์เซอร์
+GenericName[tr]=Web Tarayıcı
+GenericName[uk]=Навігатор Тенет瀏覽器
 Comment=A keyboard-driven, vim-like browser based on PyQt5
+Comment[it]= Un browser web vim-like utilizzabile da tastiera basato su PyQt5
 Icon=qutebrowser
 Type=Application
 Categories=Network;WebBrowser;
@@ -10,3 +48,128 @@ Terminal=false
 StartupNotify=false
 MimeType=text/html;text/xml;application/xhtml+xml;application/xml;application/rdf+xml;image/gif;image/jpeg;image/png;x-scheme-handler/http;x-scheme-handler/https;x-scheme-handler/qute;
 Keywords=Browser
+
+[Desktop Action new-window]
+Name=New Window
+Name[am]=አዲስ መስኮት
+Name[ar]=ﻥﺎﻓﺫﺓ ﺝﺪﻳﺩﺓ
+Name[bg]=Нов прозорец
+Name[bn]=নতুন উইন্ডো
+Name[ca]=Finestra nova
+Name[cs]=Nové okno
+Name[da]=Nyt vindue
+Name[de]=Neues Fenster
+Name[el]=Νέο Παράθυρο
+Name[en_GB]=New Window
+Name[es]=Nueva ventana
+Name[et]=Uus aken
+Name[fa]=پﻦﺟﺮﻫ ﺝﺩیﺩ
+Name[fi]=Uusi ikkuna
+Name[fil]=New Window
+Name[fr]=Nouvelle fenêtre
+Name[gu]=નવી વિંડો
+Name[hi]=नई विंडो
+Name[hr]=Novi prozor
+Name[hu]=Új ablak
+Name[id]=Jendela Baru
+Name[it]=Nuova finestra
+Name[iw]=חלון חדש
+Name[ja]=新規ウインドウ
+Name[kn]=ಹೊಸ ವಿಂಡೊ
+Name[ko]=새 창
+Name[lt]=Naujas langas
+Name[lv]=Jauns logs
+Name[ml]=പുതിയ വിന്<200d>ഡോ
+Name[mr]=नवीन विंडो
+Name[nl]=Nieuw venster
+Name[no]=Nytt vindu
+Name[pl]=Nowe okno
+Name[pt]=Nova janela
+Name[pt_BR]=Nova janela
+Name[ro]=Fereastră nouă
+Name[ru]=Новое окно
+Name[sk]=Nové okno
+Name[sl]=Novo okno
+Name[sr]=Нови прозор
+Name[sv]=Nytt fönster
+Name[sw]=Dirisha Jipya
+Name[ta]=புதிய சாளரம்
+Name[te]=క్రొత్త విండో
+Name[th]=หน้าต่างใหม่
+Name[tr]=Yeni Pencere
+Name[uk]=Нове вікно
+Name[vi]=Cửa sổ Mới
+Exec=qutebrowser
+
+[Desktop Action preferences]
+Name=Preferences
+Name[an]=Preferencias
+Name[ar]=ﺎﻠﺘﻔﻀﻳﻼﺗ
+Name[as]=পছন্দসমূহ
+Name[be]=Настройкі
+Name[bg]=Настройки
+Name[bn_IN]=পছন্দ
+Name[bs]=Postavke
+Name[ca]=Preferències
+Name[ca@valencia]=Preferències
+Name[cs]=Předvolby
+Name[da]=Indstillinger
+Name[de]=Einstellungen
+Name[el]=Προτιμήσεις
+Name[en_GB]=Preferences
+Name[eo]=Agordoj
+Name[es]=Preferencias
+Name[et]=Eelistused
+Name[eu]=Hobespenak
+Name[fa]=ﺕﺮﺟیﺡﺎﺗ
+Name[fi]=Asetukset
+Name[fr]=Préférences
+Name[fur]=Preferencis
+Name[ga]=Sainroghanna
+Name[gd]=Roghainnean
+Name[gl]=Preferencias
+Name[gu]=પસંદગીઓ
+Name[he]=העדפות
+Name[hi]=वरीयताएँ
+Name[hr]=Osobitosti
+Name[hu]=Beállítások
+Name[id]=Preferensi
+Name[is]=Kjörstillingar
+Name[it]=Preferenze
+Name[ja]=設定
+Name[kk]=Баптаулар
+Name[km]=ចំណូលចិត្ត
+Name[kn]=ಆದ್ಯತೆಗಳು
+Name[ko]=기본 설정
+Name[lt]=Nuostatos
+Name[lv]=Iestatījumi
+Name[ml]=മുന്<200d>ഗണനകള്<200d>
+Name[mr]=पसंती
+Name[nb]=Brukervalg
+Name[ne]=प्राथमिकताहरू
+Name[nl]=Voorkeuren
+Name[oc]=Preferéncias
+Name[or]=ପସନ୍ଦ
+Name[pa]=ਮੇਰੀ ਪਸੰਦ
+Name[pl]=Preferencje
+Name[pt]=Preferências
+Name[pt_BR]=Preferências
+Name[ro]=Preferințe
+Name[ru]=Параметры
+Name[sk]=Nastavenia
+Name[sl]=Možnosti
+Name[sr]=Поставке
+Name[sr@latin]=Postavke
+Name[sv]=Inställningar
+Name[ta]=விருப்பங்கள்
+Name[te]=అభీష్టాలు
+Name[tg]=Хусусиятҳо
+Name[th]=ปรับแต่ง
+Name[tr]=Tercihler
+Name[ug]=ﻡﺎﻳﻰﻠﻟﻰﻗ
+Name[uk]=Параметри
+Name[vi]=Tùy thích
+Name[zh_CN]=首选项
+Name[zh_HK]=偏好設定
+Name[zh_TW]=偏好設定
+Exec=qutebrowser "qute://settings"

misc/qutebrowser.spec

@@ -6,6 +6,8 @@ import os
 sys.path.insert(0, os.getcwd())
 from scripts import setupcommon
 
+from qutebrowser.extensions import loader
+
 block_cipher = None
 
 
@@ -27,6 +29,13 @@ def get_data_files():
     return data_files
 
 
+def get_hidden_imports():
+    imports = ['PyQt5.QtOpenGL', 'PyQt5._QOpenGLFunctions_2_0']
+    for info in loader.walk_components():
+        imports.append(info.name)
+    return imports
+
+
 setupcommon.write_git_file()
 
 
@@ -42,7 +51,7 @@ a = Analysis(['../qutebrowser/__main__.py'],
              pathex=['misc'],
              binaries=None,
              datas=get_data_files(),
-             hiddenimports=['PyQt5.QtOpenGL', 'PyQt5._QOpenGLFunctions_2_0'],
+             hiddenimports=get_hidden_imports(),
              hookspath=[],
              runtime_hooks=[],
              excludes=['tkinter'],

misc/requirements/requirements-codecov.txt

@@ -1,9 +1,9 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-certifi==2018.8.24
+certifi==2018.11.29
 chardet==3.0.4
 codecov==2.0.15
-coverage==4.5.1
+coverage==4.5.2
-idna==2.7
+idna==2.8
-requests==2.19.1
+requests==2.21.0
-urllib3==1.23
+urllib3==1.24.1

misc/requirements/requirements-flake8.txt

@@ -1,17 +1,17 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 attrs==18.2.0
-flake8==3.5.0
+flake8==3.6.0
 flake8-bugbear==18.8.0
-flake8-builtins==1.4.1  # rq.filter: != 1.4.0
+flake8-builtins==1.4.1
 flake8-comprehensions==1.4.1
-flake8-copyright==0.2.0
+flake8-copyright==0.2.2
 flake8-debugger==3.1.0
 flake8-deprecated==1.3
 flake8-docstrings==1.3.0
 flake8-future-import==0.4.5
 flake8-mock==0.3
-flake8-per-file-ignores==0.6
+flake8-per-file-ignores==0.7
 flake8-polyfill==1.0.2
 flake8-string-format==0.2.3
 flake8-tidy-imports==1.1.0
@@ -19,9 +19,9 @@ flake8-tuple==0.2.13
 mccabe==0.6.1
 pathmatch==0.2.1
 pep8-naming==0.7.0
-pycodestyle==2.3.1  # rq.filter: < 2.4.0
+pycodestyle==2.4.0
-pydocstyle==2.1.1
+pydocstyle==3.0.0
 pyflakes==2.0.0
-six==1.11.0
+six==1.12.0
 snowballstemmer==1.2.1
 typing==3.6.6

misc/requirements/requirements-flake8.txt-raw

@@ -1,6 +1,6 @@
 flake8
 flake8-bugbear
-flake8-builtins!=1.4.0
+flake8-builtins
 flake8-comprehensions
 flake8-copyright
 flake8-debugger
@@ -15,9 +15,3 @@ flake8-tuple
 pep8-naming
 pydocstyle
 pyflakes
-
-# https://github.com/PyCQA/pycodestyle/issues/741
-#@ filter: pycodestyle < 2.4.0
-
-# https://github.com/gforcada/flake8-builtins/issues/36
-#@ filter: flake8-builtins != 1.4.0

misc/requirements/requirements-mypy.txt

@@ -0,0 +1,8 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+mypy==0.650
+mypy-extensions==0.4.1
+PyQt5==5.11.3
+PyQt5-sip==4.19.13
+-e git+https://github.com/qutebrowser/PyQt5-stubs.git@wip#egg=PyQt5_stubs
+typed-ast==1.1.1

misc/requirements/requirements-mypy.txt-raw

@@ -0,0 +1,5 @@
+mypy
+-e git+https://github.com/qutebrowser/PyQt5-stubs.git@wip#egg=PyQt5-stubs
+
+# remove @commit-id for scm installs
+#@ replace: @.*# @wip#

misc/requirements/requirements-optional.txt

@@ -0,0 +1,7 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+colorama==0.4.1
+cssutils==1.0.2
+hunter==2.1.0
+Pympler==0.6
+six==1.12.0

misc/requirements/requirements-optional.txt-raw

@@ -0,0 +1,3 @@
+hunter
+cssutils
+pympler

misc/requirements/requirements-pip.txt

@@ -2,7 +2,7 @@
 
 appdirs==1.4.3
 packaging==18.0
-pyparsing==2.2.2
+pyparsing==2.3.0
-setuptools==40.4.3
+setuptools==40.6.3
-six==1.11.0
+six==1.12.0
-wheel==0.32.1
+wheel==0.32.3

misc/requirements/requirements-pyinstaller.txt

@@ -1,7 +1,7 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 altgraph==0.16.1
-future==0.16.0
+future==0.17.1
 macholib==1.11
 pefile==2018.8.8
 -e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller

misc/requirements/requirements-pylint.txt

@@ -1,23 +1,23 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 asn1crypto==0.24.0
-astroid==2.0.4
+astroid==2.1.0
-certifi==2018.8.24
+certifi==2018.11.29
 cffi==1.11.5
 chardet==3.0.4
-cryptography==2.3.1
+cryptography==2.4.2
 github3.py==1.2.0
-idna==2.7
+idna==2.8
 isort==4.3.4
-jwcrypto==0.5.0
+jwcrypto==0.6.0
 lazy-object-proxy==1.3.1
 mccabe==0.6.1
 pycparser==2.19
-pylint==2.1.1
+pylint==2.2.2
-python-dateutil==2.7.3
+python-dateutil==2.7.5
 ./scripts/dev/pylint_checkers
-requests==2.19.1
+requests==2.21.0
-six==1.11.0
+six==1.12.0
 uritemplate==3.0.0
-urllib3==1.23
+urllib3==1.24.1
 wrapt==1.10.11

misc/requirements/requirements-sphinx.txt

@@ -0,0 +1,21 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+alabaster==0.7.12
+Babel==2.6.0
+certifi==2018.11.29
+chardet==3.0.4
+docutils==0.14
+idna==2.8
+imagesize==1.1.0
+Jinja2==2.10
+MarkupSafe==1.1.0
+packaging==18.0
+Pygments==2.3.1
+pyparsing==2.3.0
+pytz==2018.7
+requests==2.21.0
+six==1.12.0
+snowballstemmer==1.2.1
+Sphinx==1.8.3
+sphinxcontrib-websupport==1.1.0
+urllib3==1.24.1

misc/requirements/requirements-sphinx.txt-raw

@@ -0,0 +1 @@
+sphinx

misc/requirements/requirements-tests.txt

@@ -3,40 +3,39 @@
 atomicwrites==1.2.1
 attrs==18.2.0
 backports.functools-lru-cache==1.5
-beautifulsoup4==4.6.3
+beautifulsoup4==4.7.0
-cheroot==6.5.2
+cheroot==6.5.3
-click==7.0
+Click==7.0
-# colorama==0.3.9
+# colorama==0.4.1
-coverage==4.5.1
+coverage==4.5.2
-EasyProcess==0.2.3
+EasyProcess==0.2.5
-fields==5.0.0
 Flask==1.0.2
 glob2==0.6
-hunter==2.0.2
+hunter==2.1.0
-hypothesis==3.74.3
+hypothesis==3.85.2
-itsdangerous==0.24
+itsdangerous==1.1.0
 # Jinja2==2.10
 Mako==1.0.7
-# MarkupSafe==1.0
+# MarkupSafe==1.1.0
-more-itertools==4.3.0
+more-itertools==5.0.0
 parse==1.9.0
 parse-type==0.4.2
-pluggy==0.7.1
+pluggy==0.8.0
-py==1.6.0
+py==1.7.0
 py-cpuinfo==4.0.0
-pytest==3.6.4  # rq.filter: <3.7
+pytest==4.0.2
-pytest-bdd==2.21.0
+pytest-bdd==3.0.1
 pytest-benchmark==3.1.1
 pytest-cov==2.6.0
 pytest-faulthandler==1.5.0
 pytest-instafail==0.4.0
 pytest-mock==1.10.0
-pytest-qt==3.2.1
+pytest-qt==3.2.2
 pytest-repeat==0.7.0
-pytest-rerunfailures==4.2
+pytest-rerunfailures==5.0
 pytest-travis-fold==1.3.0
 pytest-xvfb==1.1.0
 PyVirtualDisplay==0.2.1
-six==1.11.0
+six==1.12.0
-vulture==0.29
+vulture==1.0
 Werkzeug==0.14.1

misc/requirements/requirements-tests.txt-raw

@@ -4,7 +4,7 @@ coverage
 Flask
 hunter
 hypothesis
-pytest<3.7
+pytest
 pytest-bdd
 pytest-benchmark
 pytest-cov
@@ -19,4 +19,3 @@ pytest-xvfb
 vulture
 
 #@ ignore: Jinja2, MarkupSafe, colorama
-#@ filter: pytest <3.7

misc/requirements/requirements-tox.txt

@@ -1,8 +1,9 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-pluggy==0.7.1
+filelock==3.0.10
-py==1.6.0
+pluggy==0.8.0
-six==1.11.0
+py==1.7.0
+six==1.12.0
 toml==0.10.0
-tox==3.5.1
+tox==3.6.1
-virtualenv==16.0.0
+virtualenv==16.1.0

misc/requirements/requirements-vulture.txt

@@ -1,3 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-vulture==0.29
+vulture==1.0

misc/userscripts/README.md

@@ -53,9 +53,10 @@ The following userscripts can be found on their own repositories.
 - [qtb.us](https://github.com/Chinggis6/qtb.us): small pack of userscripts.
 - [pinboard.zsh](https://github.com/dmix/pinboard.zsh): Add URL to your
   [Pinboard][] bookmark manager.
+- [qute-capture](https://github.com/alcah/qute-capture): Capture links with
+  Emacs's org-mode to a read-later file.
 
 [Zotero]: https://www.zotero.org/
 [Pocket]: https://getpocket.com/
 [Instapaper]: https://www.instapaper.com/
 [Pinboard]: https://pinboard.in/
-

misc/userscripts/password_fill

@@ -64,6 +64,7 @@ die() {
 javascript_escape() {
     # print the first argument in an escaped way, such that it can safely
     # be used within javascripts double quotes
+    # shellcheck disable=SC2001
     sed "s,[\\\\'\"],\\\\&,g" <<< "$1"
 }
 
@@ -111,6 +112,7 @@ simplify_url() {
 # are found:
 no_entries_found() {
     while [ 0 -eq "${#files[@]}" ] && [ -n "$simple_url" ]; do
+        # shellcheck disable=SC2001
         shorter_simple_url=$(sed 's,^[^.]*\.,,' <<< "$simple_url")
         if [ "$shorter_simple_url" = "$simple_url" ] ; then
             # if no dot, then even remove the top level domain
@@ -269,7 +271,7 @@ pass_backend() {
                     break
                 fi
             fi
-        done < <($GPG "${GPG_OPTS[@]}" -d "$path" )
+        done < <($GPG "${GPG_OPTS[@]}" -d "$path" | awk 1 )
     }
 }
 # =======================================================

misc/userscripts/qute-pass

@@ -97,13 +97,19 @@ def qute_command(command):
 def find_pass_candidates(domain, password_store_path):
     candidates = []
     for path, directories, file_names in os.walk(password_store_path, followlinks=True):
-        if directories or domain not in path.split(os.path.sep):
+        secrets = fnmatch.filter(file_names, '*.gpg')
+        if not secrets:
             continue
 
         # Strip password store path prefix to get the relative pass path
         pass_path = path[len(password_store_path) + 1:]
-        secrets = fnmatch.filter(file_names, '*.gpg')
+        split_path = pass_path.split(os.path.sep)
-        candidates.extend(os.path.join(pass_path, os.path.splitext(secret)[0]) for secret in secrets)
+        for secret in secrets:
+            secret_base = os.path.splitext(secret)[0]
+            if domain not in (split_path + [secret_base]):
+                continue
+
+            candidates.append(os.path.join(pass_path, secret_base))
     return candidates
 
 

misc/userscripts/qutedmenu

@@ -37,7 +37,7 @@ get_selection() {
 # https://github.com/halfwit/dotfiles/blob/master/.config/dmenu/font
 [[ -s $confdir/dmenu/font ]] && read -r font < "$confdir"/dmenu/font
 
-[[ $font ]] && opts+=(-fn "$font")
+[[ -n $font ]] && opts+=(-fn "$font")
 
 # shellcheck source=/dev/null
 [[ -s $optsfile ]] && source "$optsfile"
@@ -46,7 +46,7 @@ url=$(get_selection)
 url=${url/*http/http}
 
 # If no selection is made, exit (escape pressed, e.g.)
-[[ ! $url ]] && exit 0
+[[ -z $url ]] && exit 0
 
 case $1 in
     open) printf '%s' "open $url" >> "$QUTE_FIFO" || qutebrowser "$url" ;;

mypy.ini

@@ -0,0 +1,87 @@
+[mypy]
+# We also need to support 3.5, but if we'd chose that here, we'd need to deal
+# with conditional imports (like secrets.py).
+python_version = 3.6
+
+# --strict
+warn_redundant_casts = True
+warn_unused_ignores = True
+disallow_subclassing_any = True
+disallow_untyped_decorators = True
+## https://github.com/python/mypy/issues/5957
+# warn_unused_configs = True
+# disallow_untyped_calls = True
+# disallow_untyped_defs = True
+## https://github.com/python/mypy/issues/5954
+# disallow_incomplete_defs = True
+# check_untyped_defs = True
+# no_implicit_optional = True
+# warn_return_any = True
+
+[mypy-colorama]
+# https://github.com/tartley/colorama/issues/206
+ignore_missing_imports = True
+
+[mypy-hunter]
+# https://github.com/ionelmc/python-hunter/issues/43
+ignore_missing_imports = True
+
+[mypy-pygments.*]
+# https://bitbucket.org/birkenfeld/pygments-main/issues/1485/type-hints
+ignore_missing_imports = True
+
+[mypy-cssutils]
+# Pretty much inactive currently
+ignore_missing_imports = True
+
+[mypy-pypeg2]
+# Pretty much inactive currently
+ignore_missing_imports = True
+
+[mypy-bdb]
+# stdlib, missing in typeshed
+ignore_missing_imports = True
+
+[mypy-qutebrowser.browser.webkit.rfc6266]
+# subclasses dynamic PyPEG2 classes
+disallow_subclassing_any = False
+
+[mypy-qutebrowser.browser.browsertab]
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+
+[mypy-qutebrowser.misc.objects]
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+
+[mypy-qutebrowser.commands.cmdutils]
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+
+[mypy-qutebrowser.config.*]
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+
+[mypy-qutebrowser.api.*]
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+
+[mypy-qutebrowser.components.*]
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+
+[mypy-qutebrowser.extensions.*]
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+
+[mypy-qutebrowser.browser.webelem]
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+
+[mypy-qutebrowser.browser.webkit.webkitelem]
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+
+[mypy-qutebrowser.browser.webengine.webengineelem]
+disallow_untyped_defs = True
+disallow_incomplete_defs = True

pytest.ini

@@ -1,6 +1,6 @@
 [pytest]
 log_level = NOTSET
-addopts = --strict -rfEw --faulthandler-timeout=90 --instafail --pythonwarnings error --benchmark-columns=Min,Max,Median
+addopts = --strict -rfEw --faulthandler-timeout=90 --instafail --benchmark-columns=Min,Max,Median
 testpaths = tests
 markers =
     gui: Tests using the GUI (e.g. spawning widgets)
@@ -64,7 +64,11 @@ qt_log_ignore =
     ^QSettings::value: Empty key passed
     ^Icon theme ".*" not found
     ^Error receiving trust for a CA certificate
+    ^QBackingStore::endPaint\(\) called with active painter on backingstore paint device
 xfail_strict = true
 filterwarnings =
+    error
     # This happens in many qutebrowser dependencies...
     ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working:DeprecationWarning
+    # WORKAROUND for https://github.com/ionelmc/pytest-benchmark/issues/124
+    ignore:Node\.warn\(code, message\) form has been deprecated, use Node\.warn\(warning_instance\) instead:pytest.PytestDeprecationWarning

qutebrowser/__init__.py

@@ -26,7 +26,7 @@ __copyright__ = "Copyright 2014-2018 Florian Bruhin (The Compiler)"
 __license__ = "GPL"
 __maintainer__ = __author__
 __email__ = "mail@qutebrowser.org"
-__version_info__ = (1, 5, 1)
+__version_info__ = (1, 5, 2)
 __version__ = '.'.join(str(e) for e in __version_info__)
 __description__ = "A keyboard-driven, vim-like browser based on PyQt5."
 

qutebrowser/api/__init__.py

@@ -0,0 +1,26 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""API for extensions.
+
+This API currently isn't exposed to third-party extensions yet, but will be in
+the future. Thus, care must be taken when adding new APIs here.
+
+Code in qutebrowser.components only uses this API.
+"""

qutebrowser/api/apitypes.py

@@ -0,0 +1,27 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""A single tab."""
+
+# pylint: disable=unused-import
+from qutebrowser.browser.browsertab import WebTabError, AbstractTab as Tab
+from qutebrowser.browser.webelem import (Error as WebElemError,
+                                         AbstractWebElement as WebElement)
+from qutebrowser.utils.usertypes import ClickTarget, JsWorld
+from qutebrowser.extensions.loader import InitContext

qutebrowser/api/cmdutils.py

@@ -0,0 +1,219 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""qutebrowser has the concept of functions, exposed to the user as commands.
+
+Creating a new command is straightforward::
+
+  from qutebrowser.api import cmdutils
+
+  @cmdutils.register(...)
+  def foo():
+      ...
+
+The commands arguments are automatically deduced by inspecting your function.
+
+The types of the function arguments are inferred based on their default values,
+e.g., an argument `foo=True` will be converted to a flag `-f`/`--foo` in
+qutebrowser's commandline.
+
+The type can be overridden using Python's function annotations::
+
+  @cmdutils.register(...)
+  def foo(bar: int, baz=True):
+      ...
+
+Possible values:
+
+- A callable (``int``, ``float``, etc.): Gets called to validate/convert the
+  value.
+- A python enum type: All members of the enum are possible values.
+- A ``typing.Union`` of multiple types above: Any of these types are valid
+  values, e.g., ``typing.Union[str, int]``.
+"""
+
+
+import inspect
+import typing
+
+from qutebrowser.utils import qtutils
+from qutebrowser.commands import command, cmdexc
+# pylint: disable=unused-import
+from qutebrowser.utils.usertypes import KeyMode, CommandValue as Value
+
+
+class CommandError(cmdexc.Error):
+
+    """Raised when a command encounters an error while running.
+
+    If your command handler encounters an error and cannot continue, raise this
+    exception with an appropriate error message::
+
+        raise cmdexc.CommandError("Message")
+
+    The message will then be shown in the qutebrowser status bar.
+
+    .. note::
+
+       You should only raise this exception while a command handler is run.
+       Raising it at another point causes qutebrowser to crash due to an
+       unhandled exception.
+    """
+
+
+def check_overflow(arg: int, ctype: str) -> None:
+    """Check if the given argument is in bounds for the given type.
+
+    Args:
+        arg: The argument to check.
+        ctype: The C++/Qt type to check as a string ('int'/'int64').
+    """
+    try:
+        qtutils.check_overflow(arg, ctype)
+    except OverflowError:
+        raise CommandError("Numeric argument is too large for internal {} "
+                           "representation.".format(ctype))
+
+
+def check_exclusive(flags: typing.Iterable[bool],
+                    names: typing.Iterable[str]) -> None:
+    """Check if only one flag is set with exclusive flags.
+
+    Raise a CommandError if not.
+
+    Args:
+        flags: The flag values to check.
+        names: A list of names (corresponding to the flags argument).
+    """
+    if sum(1 for e in flags if e) > 1:
+        argstr = '/'.join('-' + e for e in names)
+        raise CommandError("Only one of {} can be given!".format(argstr))
+
+
+class register:  # noqa: N801,N806 pylint: disable=invalid-name
+
+    """Decorator to register a new command handler."""
+
+    def __init__(self, *,
+                 instance: str = None,
+                 name: str = None,
+                 **kwargs: typing.Any) -> None:
+        """Save decorator arguments.
+
+        Gets called on parse-time with the decorator arguments.
+
+        Args:
+            See class attributes.
+        """
+        # The object from the object registry to be used as "self".
+        self._instance = instance
+        # The name (as string) or names (as list) of the command.
+        self._name = name
+        # The arguments to pass to Command.
+        self._kwargs = kwargs
+
+    def __call__(self, func: typing.Callable) -> typing.Callable:
+        """Register the command before running the function.
+
+        Gets called when a function should be decorated.
+
+        Doesn't actually decorate anything, but creates a Command object and
+        registers it in the global commands dict.
+
+        Args:
+            func: The function to be decorated.
+
+        Return:
+            The original function (unmodified).
+        """
+        if self._name is None:
+            name = func.__name__.lower().replace('_', '-')
+        else:
+            assert isinstance(self._name, str), self._name
+            name = self._name
+
+        cmd = command.Command(name=name, instance=self._instance,
+                              handler=func, **self._kwargs)
+        cmd.register()
+        return func
+
+
+class argument:  # noqa: N801,N806 pylint: disable=invalid-name
+
+    """Decorator to customize an argument.
+
+    You can customize how an argument is handled using the
+    ``@cmdutils.argument`` decorator *after* ``@cmdutils.register``. This can,
+    for example, be used to customize the flag an argument should get::
+
+      @cmdutils.register(...)
+      @cmdutils.argument('bar', flag='c')
+      def foo(bar):
+          ...
+
+    For a ``str`` argument, you can restrict the allowed strings using
+    ``choices``::
+
+      @cmdutils.register(...)
+      @cmdutils.argument('bar', choices=['val1', 'val2'])
+      def foo(bar: str):
+          ...
+
+    For ``typing.Union`` types, the given ``choices`` are only checked if other
+    types (like ``int``) don't match.
+
+    The following arguments are supported for ``@cmdutils.argument``:
+
+    - ``flag``: Customize the short flag (``-x``) the argument will get.
+    - ``value``: Tell qutebrowser to fill the argument with special values:
+
+      * ``value=cmdutils.Value.count``: The ``count`` given by the user to the
+        command.
+      * ``value=cmdutils.Value.win_id``: The window ID of the current window.
+      * ``value=cmdutils.Value.cur_tab``: The tab object which is currently
+        focused.
+
+    - ``completion``: A completion function to use when completing arguments
+      for the given command.
+    - ``choices``: The allowed string choices for the argument.
+
+    The name of an argument will always be the parameter name, with any
+    trailing underscores stripped and underscores replaced by dashes.
+    """
+
+    def __init__(self, argname: str, **kwargs: typing.Any) -> None:
+        self._argname = argname   # The name of the argument to handle.
+        self._kwargs = kwargs  # Valid ArgInfo members.
+
+    def __call__(self, func: typing.Callable) -> typing.Callable:
+        funcname = func.__name__
+
+        if self._argname not in inspect.signature(func).parameters:
+            raise ValueError("{} has no argument {}!".format(funcname,
+                                                             self._argname))
+        if not hasattr(func, 'qute_args'):
+            func.qute_args = {}  # type: ignore
+        elif func.qute_args is None:  # type: ignore
+            raise ValueError("@cmdutils.argument got called above (after) "
+                             "@cmdutils.register for {}!".format(funcname))
+
+        arginfo = command.ArgInfo(**self._kwargs)
+        func.qute_args[self._argname] = arginfo  # type: ignore
+
+        return func

qutebrowser/api/config.py

@@ -0,0 +1,43 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Access to the qutebrowser configuration."""
+
+import typing
+
+from PyQt5.QtCore import QUrl
+
+from qutebrowser.config import config
+
+#: Simplified access to config values using attribute acccess.
+#: For example, to access the ``content.javascript.enabled`` setting,
+#: you can do::
+#:
+#:   if config.val.content.javascript.enabled:
+#:       ...
+#:
+#: This also supports setting configuration values::
+#:
+#:   config.val.content.javascript.enabled = False
+val = typing.cast('config.ConfigContainer', None)
+
+
+def get(name: str, url: QUrl = None) -> typing.Any:
+    """Get a value from the config based on a string name."""
+    return config.instance.get(name, url)

qutebrowser/api/downloads.py

@@ -0,0 +1,75 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+
+"""APIs related to downloading files."""
+
+
+import io
+
+from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot, QUrl
+
+from qutebrowser.browser import downloads, qtnetworkdownloads
+from qutebrowser.utils import objreg
+
+
+class TempDownload(QObject):
+
+    """A download of some data into a file object."""
+
+    finished = pyqtSignal()
+
+    def __init__(self, item: qtnetworkdownloads.DownloadItem) -> None:
+        super().__init__()
+        self._item = item
+        self._item.finished.connect(self._on_download_finished)
+        self.successful = False
+        self.fileobj = item.fileobj
+
+    @pyqtSlot()
+    def _on_download_finished(self) -> None:
+        self.successful = self._item.successful
+        self.finished.emit()
+
+
+def download_temp(url: QUrl) -> TempDownload:
+    """Download the given URL into a file object.
+
+    The download is not saved to disk.
+
+    Returns a ``TempDownload`` object, which triggers a ``finished`` signal
+    when the download has finished::
+
+        dl = downloads.download_temp(QUrl("https://www.example.com/"))
+        dl.finished.connect(functools.partial(on_download_finished, dl))
+
+    After the download has finished, its ``successful`` attribute can be
+    checked to make sure it finished successfully. If so, its contents can be
+    read by accessing the ``fileobj`` attribute::
+
+        def on_download_finished(download: downloads.TempDownload) -> None:
+            if download.successful:
+                print(download.fileobj.read())
+                download.fileobj.close()
+    """
+    fobj = io.BytesIO()
+    fobj.name = 'temporary: ' + url.host()
+    target = downloads.FileObjDownloadTarget(fobj)
+    download_manager = objreg.get('qtnetwork-download-manager')
+    return download_manager.get(url, target=target, auto_remove=True)

qutebrowser/api/hook.py

@@ -0,0 +1,92 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+# pylint: disable=invalid-name
+
+"""Hooks for extensions."""
+
+import importlib
+import typing
+
+
+from qutebrowser.extensions import loader
+
+
+def _add_module_info(func: typing.Callable) -> loader.ModuleInfo:
+    """Add module info to the given function."""
+    module = importlib.import_module(func.__module__)
+    return loader.add_module_info(module)
+
+
+class init:
+
+    """Decorator to mark a function to run when initializing.
+
+    The decorated function gets called with a
+    :class:`qutebrowser.api.apitypes.InitContext` as argument.
+
+    Example::
+
+        @hook.init()
+        def init(_context):
+            message.info("Extension initialized.")
+    """
+
+    def __call__(self, func: typing.Callable) -> typing.Callable:
+        info = _add_module_info(func)
+        if info.init_hook is not None:
+            raise ValueError("init hook is already registered!")
+        info.init_hook = func
+        return func
+
+
+class config_changed:
+
+    """Decorator to get notified about changed configs.
+
+    By default, the decorated function is called when any change in the config
+    occurs::
+
+        @hook.config_changed()
+        def on_config_changed():
+            ...
+
+    When an option name is passed, it's only called when the given option was
+    changed::
+
+        @hook.config_changed('content.javascript.enabled')
+        def on_config_changed():
+            ...
+
+    Alternatively, a part of an option name can be specified. In the following
+    snippet, ``on_config_changed`` gets called when either
+    ``bindings.commands`` or ``bindings.key_mappings`` have changed::
+
+        @hook.config_changed('bindings')
+        def on_config_changed():
+            ...
+    """
+
+    def __init__(self, option_filter: str = None) -> None:
+        self._filter = option_filter
+
+    def __call__(self, func: typing.Callable) -> typing.Callable:
+        info = _add_module_info(func)
+        info.config_changed_hooks.append((self._filter, func))
+        return func

qutebrowser/api/interceptor.py

@@ -0,0 +1,43 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""APIs related to intercepting/blocking requests."""
+
+from qutebrowser.extensions import interceptors
+# pylint: disable=unused-import
+from qutebrowser.extensions.interceptors import Request
+
+
+#: Type annotation for an interceptor function.
+InterceptorType = interceptors.InterceptorType
+
+
+def register(interceptor: InterceptorType) -> None:
+    """Register a request interceptor.
+
+    Whenever a request happens, the interceptor gets called with a
+    :class:`Request` object.
+
+    Example::
+
+        def intercept(request: interceptor.Request) -> None:
+            if request.request_url.host() == 'badhost.example.com':
+                request.block()
+    """
+    interceptors.register(interceptor)

qutebrowser/api/message.py

@@ -0,0 +1,23 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Utilities to display messages above the status bar."""
+
+# pylint: disable=unused-import
+from qutebrowser.utils.message import error, warning, info

qutebrowser/app.py

@@ -60,13 +60,15 @@ except ImportError:
 import qutebrowser
 import qutebrowser.resources
 from qutebrowser.completion.models import miscmodels
-from qutebrowser.commands import cmdutils, runners, cmdexc
+from qutebrowser.commands import runners
+from qutebrowser.api import cmdutils
 from qutebrowser.config import config, websettings, configfiles, configinit
-from qutebrowser.browser import (urlmarks, adblock, history, browsertab,
+from qutebrowser.browser import (urlmarks, history, browsertab,
                                  qtnetworkdownloads, downloads, greasemonkey)
 from qutebrowser.browser.network import proxy
 from qutebrowser.browser.webkit import cookies, cache
 from qutebrowser.browser.webkit.network import networkmanager
+from qutebrowser.extensions import loader
 from qutebrowser.keyinput import macros
 from qutebrowser.mainwindow import mainwindow, prompt
 from qutebrowser.misc import (readline, ipc, savemanager, sessions,
@@ -163,6 +165,8 @@ def init(args, crash_handler):
     qApp.setQuitOnLastWindowClosed(False)
     _init_icon()
 
+    loader.init()
+    loader.load_components()
     try:
         _init_modules(args, crash_handler)
     except (OSError, UnicodeDecodeError, browsertab.WebTabError) as e:
@@ -193,7 +197,7 @@ def _init_icon():
     icon = QIcon()
     fallback_icon = QIcon()
     for size in [16, 24, 32, 48, 64, 96, 128, 256, 512]:
-        filename = ':/icons/qutebrowser-{}x{}.png'.format(size, size)
+        filename = ':/icons/qutebrowser-{size}x{size}.png'.format(size=size)
         pixmap = QPixmap(filename)
         if pixmap.isNull():
             log.init.warning("Failed to load {}".format(filename))
@@ -303,10 +307,10 @@ def process_pos_args(args, via_ipc=False, cwd=None, target_arg=None):
 
 
 def open_url(url, target=None, no_raise=False, via_ipc=True):
-    """Open an URL in new window/tab.
+    """Open a URL in new window/tab.
 
     Args:
-        url: An URL to open.
+        url: A URL to open.
         target: same as new_instance_open_target (used as a default).
         no_raise: suppress target window raising.
         via_ipc: Whether the arguments were transmitted over IPC.
@@ -465,11 +469,6 @@ def _init_modules(args, crash_handler):
     log.init.debug("Initializing websettings...")
     websettings.init(args)
 
-    log.init.debug("Initializing adblock...")
-    host_blocker = adblock.HostBlocker()
-    host_blocker.read_hosts()
-    objreg.register('host-blocker', host_blocker)
-
     log.init.debug("Initializing quickmarks...")
     quickmark_manager = urlmarks.QuickmarkManager(qApp)
     objreg.register('quickmark-manager', quickmark_manager)
@@ -619,10 +618,11 @@ class Quitter:
             ok = self.restart(session='_restart')
         except sessions.SessionError as e:
             log.destroy.exception("Failed to save session!")
-            raise cmdexc.CommandError("Failed to save session: {}!".format(e))
+            raise cmdutils.CommandError("Failed to save session: {}!"
+                                        .format(e))
         except SyntaxError as e:
             log.destroy.exception("Got SyntaxError")
-            raise cmdexc.CommandError("SyntaxError in {}:{}: {}".format(
+            raise cmdutils.CommandError("SyntaxError in {}:{}: {}".format(
                 e.filename, e.lineno, e))
         if ok:
             self.shutdown(restart=True)
@@ -684,7 +684,7 @@ class Quitter:
             session: The name of the session to save.
         """
         if session is not None and not save:
-            raise cmdexc.CommandError("Session name given without --save!")
+            raise cmdutils.CommandError("Session name given without --save!")
         if save:
             if session is None:
                 session = sessions.default

qutebrowser/browser/downloads.py

@@ -33,14 +33,18 @@ from PyQt5.QtCore import (pyqtSlot, pyqtSignal, Qt, QObject, QModelIndex,
                           QTimer, QAbstractListModel, QUrl)
 
 from qutebrowser.browser import pdfjs
-from qutebrowser.commands import cmdexc, cmdutils
+from qutebrowser.api import cmdutils
 from qutebrowser.config import config
 from qutebrowser.utils import (usertypes, standarddir, utils, message, log,
                                qtutils, objreg)
 from qutebrowser.qt import sip
 
 
-ModelRole = enum.IntEnum('ModelRole', ['item'], start=Qt.UserRole)
+class ModelRole(enum.IntEnum):
+
+    """Custom download model roles."""
+
+    item = Qt.UserRole
 
 
 # Remember the last used directory
@@ -60,8 +64,6 @@ class UnsupportedAttribute:
     supported with QtWebengine.
     """
 
-    pass
-
 
 class UnsupportedOperationError(Exception):
 
@@ -1007,11 +1009,11 @@ class DownloadModel(QAbstractListModel):
             count: The index of the download
         """
         if not count:
-            raise cmdexc.CommandError("There's no download!")
+            raise cmdutils.CommandError("There's no download!")
-        raise cmdexc.CommandError("There's no download {}!".format(count))
+        raise cmdutils.CommandError("There's no download {}!".format(count))
 
     @cmdutils.register(instance='download-model', scope='window')
-    @cmdutils.argument('count', count=True)
+    @cmdutils.argument('count', value=cmdutils.Value.count)
     def download_cancel(self, all_=False, count=0):
         """Cancel the last/[count]th download.
 
@@ -1032,12 +1034,12 @@ class DownloadModel(QAbstractListModel):
             if download.done:
                 if not count:
                     count = len(self)
-                raise cmdexc.CommandError("Download {} is already done!"
+                raise cmdutils.CommandError("Download {} is already done!"
-                                          .format(count))
+                                            .format(count))
             download.cancel()
 
     @cmdutils.register(instance='download-model', scope='window')
-    @cmdutils.argument('count', count=True)
+    @cmdutils.argument('count', value=cmdutils.Value.count)
     def download_delete(self, count=0):
         """Delete the last/[count]th download from disk.
 
@@ -1051,14 +1053,15 @@ class DownloadModel(QAbstractListModel):
         if not download.successful:
             if not count:
                 count = len(self)
-            raise cmdexc.CommandError("Download {} is not done!".format(count))
+            raise cmdutils.CommandError("Download {} is not done!"
+                                        .format(count))
         download.delete()
         download.remove()
         log.downloads.debug("deleted download {}".format(download))
 
     @cmdutils.register(instance='download-model', scope='window', maxsplit=0)
-    @cmdutils.argument('count', count=True)
+    @cmdutils.argument('count', value=cmdutils.Value.count)
-    def download_open(self, cmdline: str = None, count=0):
+    def download_open(self, cmdline: str = None, count: int = 0) -> None:
         """Open the last/[count]th download.
 
         If no specific command is given, this will use the system's default
@@ -1078,11 +1081,12 @@ class DownloadModel(QAbstractListModel):
         if not download.successful:
             if not count:
                 count = len(self)
-            raise cmdexc.CommandError("Download {} is not done!".format(count))
+            raise cmdutils.CommandError("Download {} is not done!"
+                                        .format(count))
         download.open_file(cmdline)
 
     @cmdutils.register(instance='download-model', scope='window')
-    @cmdutils.argument('count', count=True)
+    @cmdutils.argument('count', value=cmdutils.Value.count)
     def download_retry(self, count=0):
         """Retry the first failed/[count]th download.
 
@@ -1095,12 +1099,12 @@ class DownloadModel(QAbstractListModel):
             except IndexError:
                 self._raise_no_download(count)
             if download.successful or not download.done:
-                raise cmdexc.CommandError("Download {} did not fail!".format(
+                raise cmdutils.CommandError("Download {} did not fail!"
-                    count))
+                                            .format(count))
         else:
             to_retry = [d for d in self if d.done and not d.successful]
             if not to_retry:
-                raise cmdexc.CommandError("No failed downloads!")
+                raise cmdutils.CommandError("No failed downloads!")
             else:
                 download = to_retry[0]
         download.try_retry()
@@ -1117,7 +1121,7 @@ class DownloadModel(QAbstractListModel):
                 download.remove()
 
     @cmdutils.register(instance='download-model', scope='window')
-    @cmdutils.argument('count', count=True)
+    @cmdutils.argument('count', value=cmdutils.Value.count)
     def download_remove(self, all_=False, count=0):
         """Remove the last/[count]th download from the list.
 
@@ -1135,8 +1139,8 @@ class DownloadModel(QAbstractListModel):
             if not download.done:
                 if not count:
                     count = len(self)
-                raise cmdexc.CommandError("Download {} is not done!"
+                raise cmdutils.CommandError("Download {} is not done!"
-                                          .format(count))
+                                            .format(count))
             download.remove()
 
     def running_downloads(self):

qutebrowser/browser/downloadview.py

@@ -75,7 +75,8 @@ class DownloadView(QListView):
 
     def __init__(self, win_id, parent=None):
         super().__init__(parent)
-        self.setStyle(QStyleFactory.create('Fusion'))
+        if not utils.is_mac:
+            self.setStyle(QStyleFactory.create('Fusion'))
         config.set_register_stylesheet(self)
         self.setResizeMode(QListView.Adjust)
         self.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)

qutebrowser/browser/greasemonkey.py

@@ -32,7 +32,7 @@ from PyQt5.QtCore import pyqtSignal, QObject, QUrl
 
 from qutebrowser.utils import (log, standarddir, jinja, objreg, utils,
                                javascript, urlmatch, version, usertypes)
-from qutebrowser.commands import cmdutils
+from qutebrowser.api import cmdutils
 from qutebrowser.browser import downloads
 from qutebrowser.misc import objects
 

qutebrowser/browser/hints.py

@@ -21,7 +21,6 @@
 
 import collections
 import functools
-import math
 import os
 import re
 import html
@@ -35,7 +34,8 @@ from PyQt5.QtWidgets import QLabel
 from qutebrowser.config import config, configexc
 from qutebrowser.keyinput import modeman, modeparsers
 from qutebrowser.browser import webelem
-from qutebrowser.commands import userscripts, cmdexc, cmdutils, runners
+from qutebrowser.commands import userscripts, runners
+from qutebrowser.api import cmdutils
 from qutebrowser.utils import usertypes, log, qtutils, message, objreg, utils
 
 
@@ -102,7 +102,7 @@ class HintLabel(QLabel):
             matched: The part of the text which was typed.
             unmatched: The part of the text which was not typed yet.
         """
-        if (config.val.hints.uppercase and
+        if (config.cache['hints.uppercase'] and
                 self._context.hint_mode in ['letter', 'word']):
             matched = html.escape(matched.upper())
             unmatched = html.escape(unmatched.upper())
@@ -110,9 +110,12 @@ class HintLabel(QLabel):
             matched = html.escape(matched)
             unmatched = html.escape(unmatched)
 
-        match_color = html.escape(config.val.colors.hints.match.fg)
+        match_color = html.escape(config.cache['colors.hints.match.fg'])
-        self.setText('<font color="{}">{}</font>{}'.format(
+        if matched:
-            match_color, matched, unmatched))
+            self.setText('<font color="{}">{}</font>{}'.format(
+                match_color, matched, unmatched))
+        else:
+            self.setText(unmatched)
         self.adjustSize()
 
     @pyqtSlot()
@@ -123,7 +126,7 @@ class HintLabel(QLabel):
             log.hints.debug("Frame for {!r} vanished!".format(self))
             self.hide()
             return
-        no_js = config.val.hints.find_implementation != 'javascript'
+        no_js = config.cache['hints.find_implementation'] != 'javascript'
         rect = self.elem.rect_on_view(no_js=no_js)
         self.move(rect.x(), rect.y())
 
@@ -215,9 +218,7 @@ class HintActions:
 
         if context.target in [Target.normal, Target.current]:
             # Set the pre-jump mark ', so we can jump back here after following
-            tabbed_browser = objreg.get('tabbed-browser', scope='window',
+            context.tab.scroller.before_jump_requested.emit()
-                                        window=self._win_id)
-            tabbed_browser.set_mark("'")
 
         try:
             if context.target == Target.hover:
@@ -302,8 +303,8 @@ class HintActions:
             raise HintingError("No suitable link found for this element.")
 
         prompt = False if context.rapid else None
-        qnam = context.tab.networkaccessmanager()
+        qnam = context.tab.private_api.networkaccessmanager()
-        user_agent = context.tab.user_agent()
+        user_agent = context.tab.private_api.user_agent()
 
         # FIXME:qtwebengine do this with QtWebEngine downloads?
         download_manager = objreg.get('qtnetwork-download-manager')
@@ -454,21 +455,15 @@ class HintManager(QObject):
         # Determine how many digits the link hints will require in the worst
         # case. Usually we do not need all of these digits for every link
         # single hint, so we can show shorter hints for a few of the links.
-        needed = max(min_chars, math.ceil(math.log(len(elems), len(chars))))
+        needed = max(min_chars, utils.ceil_log(len(elems), len(chars)))
+
         # Short hints are the number of hints we can possibly show which are
         # (needed - 1) digits in length.
-        if needed > min_chars:
+        if needed > min_chars and needed > 1:
             total_space = len(chars) ** needed
-            # Calculate short_count naively, by finding the avaiable space and
+            # For each 1 short link being added, len(chars) long links are
-            # dividing by the number of spots we would loose by adding a
+            # removed, therefore the space removed is len(chars) - 1.
-            # short element
+            short_count = (total_space - len(elems)) // (len(chars) - 1)
-            short_count = math.floor((total_space - len(elems)) /
-                                     len(chars))
-            # Check if we double counted above to warrant another short_count
-            # https://github.com/qutebrowser/qutebrowser/issues/3242
-            if total_space - (short_count * len(chars) +
-                              (len(elems) - short_count)) >= len(chars) - 1:
-                short_count += 1
         else:
             short_count = 0
 
@@ -495,7 +490,7 @@ class HintManager(QObject):
             elems: The elements to generate labels for.
         """
         strings = []
-        needed = max(min_chars, math.ceil(math.log(len(elems), len(chars))))
+        needed = max(min_chars, utils.ceil_log(len(elems), len(chars)))
         for i in range(len(elems)):
             strings.append(self._number_to_hint_str(i, chars, needed))
         return strings
@@ -567,12 +562,12 @@ class HintManager(QObject):
         if target in [Target.userscript, Target.spawn, Target.run,
                       Target.fill]:
             if not args:
-                raise cmdexc.CommandError(
+                raise cmdutils.CommandError(
                     "'args' is required with target userscript/spawn/run/"
                     "fill.")
         else:
             if args:
-                raise cmdexc.CommandError(
+                raise cmdutils.CommandError(
                     "'args' is only allowed with target userscript/spawn.")
 
     def _filter_matches(self, filterstr, elemstr):
@@ -600,13 +595,6 @@ class HintManager(QObject):
             log.hints.debug("In _start_cb without context!")
             return
 
-        if elems is None:
-            message.error("Unknown error while getting hint elements.")
-            return
-        elif isinstance(elems, webelem.Error):
-            message.error(str(elems))
-            return
-
         if not elems:
             message.error("No elements found.")
             return
@@ -639,9 +627,8 @@ class HintManager(QObject):
 
     @cmdutils.register(instance='hintmanager', scope='tab', name='hint',
                        star_args_optional=True, maxsplit=2)
-    @cmdutils.argument('win_id', win_id=True)
     def start(self,  # pylint: disable=keyword-arg-before-vararg
-              group='all', target=Target.normal, *args, win_id, mode=None,
+              group='all', target=Target.normal, *args, mode=None,
               add_history=False, rapid=False, first=False):
         """Start hinting.
 
@@ -710,12 +697,12 @@ class HintManager(QObject):
                                     window=self._win_id)
         tab = tabbed_browser.widget.currentWidget()
         if tab is None:
-            raise cmdexc.CommandError("No WebView available yet!")
+            raise cmdutils.CommandError("No WebView available yet!")
 
         mode_manager = objreg.get('mode-manager', scope='window',
                                   window=self._win_id)
         if mode_manager.mode == usertypes.KeyMode.hint:
-            modeman.leave(win_id, usertypes.KeyMode.hint, 're-hinting')
+            modeman.leave(self._win_id, usertypes.KeyMode.hint, 're-hinting')
 
         if rapid:
             if target in [Target.tab_bg, Target.window, Target.run,
@@ -727,8 +714,8 @@ class HintManager(QObject):
                 pass
             else:
                 name = target.name.replace('_', '-')
-                raise cmdexc.CommandError("Rapid hinting makes no sense with "
+                raise cmdutils.CommandError("Rapid hinting makes no sense "
-                                          "target {}!".format(name))
+                                            "with target {}!".format(name))
 
         self._check_args(target, *args)
         self._context = HintContext()
@@ -741,18 +728,21 @@ class HintManager(QObject):
         try:
             self._context.baseurl = tabbed_browser.current_url()
         except qtutils.QtValueError:
-            raise cmdexc.CommandError("No URL set for this page yet!")
+            raise cmdutils.CommandError("No URL set for this page yet!")
-        self._context.args = args
+        self._context.args = list(args)
         self._context.group = group
 
         try:
             selector = webelem.css_selector(self._context.group,
                                             self._context.baseurl)
         except webelem.Error as e:
-            raise cmdexc.CommandError(str(e))
+            raise cmdutils.CommandError(str(e))
 
-        self._context.tab.elements.find_css(selector, self._start_cb,
+        self._context.tab.elements.find_css(
-                                            only_visible=True)
+            selector,
+            callback=self._start_cb,
+            error_cb=lambda err: message.error(str(err)),
+            only_visible=True)
 
     def _get_hint_mode(self, mode):
         """Get the hinting mode to use based on a mode argument."""
@@ -763,7 +753,7 @@ class HintManager(QObject):
         try:
             opt.typ.to_py(mode)
         except configexc.ValidationError as e:
-            raise cmdexc.CommandError("Invalid mode: {}".format(e))
+            raise cmdutils.CommandError("Invalid mode: {}".format(e))
         return mode
 
     def current_mode(self):
@@ -838,9 +828,10 @@ class HintManager(QObject):
 
         Args:
             filterstr: The string to filter with, or None to use the filter
-                       from previous call (saved in `self._filterstr`). If
+                       from previous call (saved in `self._context.filterstr`).
-                       `filterstr` is an empty string or if both `filterstr`
+                       If `filterstr` is an empty string or if both `filterstr`
-                       and `self._filterstr` are None, all hints are shown.
+                       and `self._context.filterstr` are None, all hints are
+                       shown.
         """
         if filterstr is None:
             filterstr = self._context.filterstr
@@ -964,13 +955,13 @@ class HintManager(QObject):
         """
         if keystring is None:
             if self._context.to_follow is None:
-                raise cmdexc.CommandError("No hint to follow")
+                raise cmdutils.CommandError("No hint to follow")
             elif select:
-                raise cmdexc.CommandError("Can't use --select without hint.")
+                raise cmdutils.CommandError("Can't use --select without hint.")
             else:
                 keystring = self._context.to_follow
         elif keystring not in self._context.labels:
-            raise cmdexc.CommandError("No hint {}!".format(keystring))
+            raise cmdutils.CommandError("No hint {}!".format(keystring))
 
         if select:
             self.handle_partial_key(keystring)

qutebrowser/browser/history.py

@@ -27,7 +27,7 @@ from PyQt5.QtCore import pyqtSlot, QUrl, pyqtSignal
 from PyQt5.QtWidgets import QProgressDialog, QApplication
 
 from qutebrowser.config import config
-from qutebrowser.commands import cmdutils, cmdexc
+from qutebrowser.api import cmdutils
 from qutebrowser.utils import utils, objreg, log, usertypes, message, qtutils
 from qutebrowser.misc import objects, sql
 
@@ -365,7 +365,8 @@ class WebHistory(sql.SqlTable):
                 f.write('\n'.join(lines))
             message.info("Dumped history to {}".format(dest))
         except OSError as e:
-            raise cmdexc.CommandError('Could not write history: {}'.format(e))
+            raise cmdutils.CommandError('Could not write history: {}'
+                                        .format(e))
 
 
 def init(parent=None):

qutebrowser/browser/inspector.py

@@ -49,8 +49,6 @@ class WebInspectorError(Exception):
 
     """Raised when the inspector could not be initialized."""
 
-    pass
-
 
 class AbstractWebInspector(QWidget):
 

qutebrowser/browser/mouse.py

@@ -240,7 +240,7 @@ class MouseEventFilter(QObject):
         evtype = event.type()
         if evtype not in self._handlers:
             return False
-        if obj is not self._tab.event_target():
+        if obj is not self._tab.private_api.event_target():
             log.mouse.debug("Ignoring {} to {}".format(
                 event.__class__.__name__, obj))
             return False

qutebrowser/browser/navigate.py

@@ -116,13 +116,6 @@ def prevnext(*, browsertab, win_id, baseurl, prev=False,
         window: True to open in a new window, False for the current one.
     """
     def _prevnext_cb(elems):
-        if elems is None:
-            message.error("Unknown error while getting hint elements")
-            return
-        elif isinstance(elems, webelem.Error):
-            message.error(str(elems))
-            return
-
         elem = _find_prevnext(prev, elems)
         word = 'prev' if prev else 'forward'
 
@@ -140,7 +133,7 @@ def prevnext(*, browsertab, win_id, baseurl, prev=False,
 
         if window:
             new_window = mainwindow.MainWindow(
-                private=cur_tabbed_browser.private)
+                private=cur_tabbed_browser.is_private)
             new_window.show()
             tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                         window=new_window.win_id)
@@ -148,11 +141,12 @@ def prevnext(*, browsertab, win_id, baseurl, prev=False,
         elif tab:
             cur_tabbed_browser.tabopen(url, background=background)
         else:
-            browsertab.openurl(url)
+            browsertab.load_url(url)
 
     try:
         link_selector = webelem.css_selector('links', baseurl)
     except webelem.Error as e:
         raise Error(str(e))
 
-    browsertab.elements.find_css(link_selector, _prevnext_cb)
+    browsertab.elements.find_css(link_selector, callback=_prevnext_cb,
+                                 error_cb=lambda err: message.error(str(err)))

qutebrowser/browser/network/pac.py

@@ -35,15 +35,11 @@ class ParseProxyError(Exception):
 
     """Error while parsing PAC result string."""
 
-    pass
-
 
 class EvalProxyError(Exception):
 
     """Error while evaluating PAC script."""
 
-    pass
-
 
 def _js_slot(*args):
     """Wrap a methods as a JavaScript function.
@@ -184,6 +180,8 @@ class PACResolver:
         """
         self._engine = QJSEngine()
 
+        self._engine.installExtensions(QJSEngine.ConsoleExtension)
+
         self._ctx = _PACContext(self._engine)
         self._engine.globalObject().setProperty(
             "PAC", self._engine.newQObject(self._ctx))

qutebrowser/browser/qutescheme.py

@@ -37,7 +37,7 @@ try:
     import secrets
 except ImportError:
     # New in Python 3.6
-    secrets = None
+    secrets = None  # type: ignore
 
 from PyQt5.QtCore import QUrlQuery, QUrl, qVersion
 
@@ -61,36 +61,26 @@ class Error(Exception):
 
     """Exception for generic errors on a qute:// page."""
 
-    pass
-
 
 class NotFoundError(Error):
 
     """Raised when the given URL was not found."""
 
-    pass
-
 
 class SchemeOSError(Error):
 
     """Raised when there was an OSError inside a handler."""
 
-    pass
-
 
 class UrlInvalidError(Error):
 
     """Raised when an invalid URL was opened."""
 
-    pass
-
 
 class RequestDeniedError(Error):
 
     """Raised when the request is forbidden."""
 
-    pass
-
 
 class Redirect(Exception):
 

qutebrowser/browser/shared.py

@@ -262,7 +262,7 @@ def get_tab(win_id, target):
     elif target == usertypes.ClickTarget.window:
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window=win_id)
-        window = mainwindow.MainWindow(private=tabbed_browser.private)
+        window = mainwindow.MainWindow(private=tabbed_browser.is_private)
         window.show()
         win_id = window.win_id
         bg_tab = False

qutebrowser/browser/urlmarks.py

@@ -35,7 +35,7 @@ from PyQt5.QtCore import pyqtSignal, QUrl, QObject
 
 from qutebrowser.utils import (message, usertypes, qtutils, urlutils,
                                standarddir, objreg, log)
-from qutebrowser.commands import cmdutils
+from qutebrowser.api import cmdutils
 from qutebrowser.misc import lineparser
 
 
@@ -43,29 +43,21 @@ class Error(Exception):
 
     """Base class for all errors in this module."""
 
-    pass
-
 
 class InvalidUrlError(Error):
 
     """Exception emitted when a URL is invalid."""
 
-    pass
-
 
 class DoesNotExistError(Error):
 
     """Exception emitted when a given URL does not exist."""
 
-    pass
-
 
 class AlreadyExistsError(Error):
 
     """Exception emitted when a given URL does already exist."""
 
-    pass
-
 
 class UrlMarkManager(QObject):
 
@@ -174,7 +166,7 @@ class QuickmarkManager(UrlMarkManager):
             url: The url to add as quickmark.
             name: The name for the new quickmark.
         """
-        # We don't raise cmdexc.CommandError here as this can be called async
+        # We don't raise cmdutils.CommandError here as this can be called async
         # via prompt_save.
         if not name:
             message.error("Can't set mark with empty name!")

qutebrowser/browser/webelem.py

@@ -19,32 +19,36 @@
 
 """Generic web element related code."""
 
+import typing
 import collections.abc
 
-from PyQt5.QtCore import QUrl, Qt, QEvent, QTimer
+from PyQt5.QtCore import QUrl, Qt, QEvent, QTimer, QRect, QPoint
 from PyQt5.QtGui import QMouseEvent
 
 from qutebrowser.config import config
 from qutebrowser.keyinput import modeman
 from qutebrowser.mainwindow import mainwindow
 from qutebrowser.utils import log, usertypes, utils, qtutils, objreg
+MYPY = False
+if MYPY:
+    # pylint: disable=unused-import,useless-suppression
+    from qutebrowser.browser import browsertab
+
+
+JsValueType = typing.Union[int, float, str, None]
 
 
 class Error(Exception):
 
     """Base class for WebElement errors."""
 
-    pass
-
 
 class OrphanedError(Error):
 
     """Raised when a webelement's parent has vanished."""
 
-    pass
 
-
+def css_selector(group: str, url: QUrl) -> str:
-def css_selector(group, url):
     """Get a CSS selector for the given group/URL."""
     selectors = config.instance.get('hints.selectors', url)
     if group not in selectors:
@@ -58,76 +62,74 @@ def css_selector(group, url):
 
 class AbstractWebElement(collections.abc.MutableMapping):
 
-    """A wrapper around QtWebKit/QtWebEngine web element.
+    """A wrapper around QtWebKit/QtWebEngine web element."""
 
-    Attributes:
+    def __init__(self, tab: 'browsertab.AbstractTab') -> None:
-        tab: The tab associated with this element.
-    """
-
-    def __init__(self, tab):
         self._tab = tab
 
-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
         raise NotImplementedError
 
-    def __str__(self):
+    def __str__(self) -> str:
         raise NotImplementedError
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> str:
         raise NotImplementedError
 
-    def __setitem__(self, key, val):
+    def __setitem__(self, key: str, val: str) -> None:
         raise NotImplementedError
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: str) -> None:
         raise NotImplementedError
 
-    def __iter__(self):
+    def __iter__(self) -> typing.Iterator[str]:
         raise NotImplementedError
 
-    def __len__(self):
+    def __len__(self) -> int:
         raise NotImplementedError
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         try:
             html = utils.compact_text(self.outer_xml(), 500)
         except Error:
             html = None
         return utils.get_repr(self, html=html)
 
-    def has_frame(self):
+    def has_frame(self) -> bool:
         """Check if this element has a valid frame attached."""
         raise NotImplementedError
 
-    def geometry(self):
+    def geometry(self) -> QRect:
         """Get the geometry for this element."""
         raise NotImplementedError
 
-    def classes(self):
+    def classes(self) -> typing.List[str]:
         """Get a list of classes assigned to this element."""
         raise NotImplementedError
 
-    def tag_name(self):
+    def tag_name(self) -> str:
         """Get the tag name of this element.
 
         The returned name will always be lower-case.
         """
         raise NotImplementedError
 
-    def outer_xml(self):
+    def outer_xml(self) -> str:
         """Get the full HTML representation of this element."""
         raise NotImplementedError
 
-    def value(self):
+    def value(self) -> JsValueType:
         """Get the value attribute for this element, or None."""
         raise NotImplementedError
 
-    def set_value(self, value):
+    def set_value(self, value: JsValueType) -> None:
         """Set the element value."""
         raise NotImplementedError
 
-    def dispatch_event(self, event, bubbles=False,
+    def dispatch_event(self, event: str,
-                       cancelable=False, composed=False):
+                       bubbles: bool = False,
+                       cancelable: bool = False,
+                       composed: bool = False) -> None:
         """Dispatch an event to the element.
 
         Args:
@@ -138,35 +140,25 @@ class AbstractWebElement(collections.abc.MutableMapping):
         """
         raise NotImplementedError
 
-    def insert_text(self, text):
+    def insert_text(self, text: str) -> None:
         """Insert the given text into the element."""
         raise NotImplementedError
 
-    def rect_on_view(self, *, elem_geometry=None, no_js=False):
+    def rect_on_view(self, *, elem_geometry: QRect = None,
+                     no_js: bool = False) -> QRect:
         """Get the geometry of the element relative to the webview.
 
-        Uses the getClientRects() JavaScript method to obtain the collection of
-        rectangles containing the element and returns the first rectangle which
-        is large enough (larger than 1px times 1px). If all rectangles returned
-        by getClientRects() are too small, falls back to elem.rect_on_view().
-
-        Skipping of small rectangles is due to <a> elements containing other
-        elements with "display:block" style, see
-        https://github.com/qutebrowser/qutebrowser/issues/1298
-
         Args:
             elem_geometry: The geometry of the element, or None.
-                           Calling QWebElement::geometry is rather expensive so
+            no_js: Fall back to the Python implementation.
-                           we want to avoid doing it twice.
-            no_js: Fall back to the Python implementation
         """
         raise NotImplementedError
 
-    def is_writable(self):
+    def is_writable(self) -> bool:
         """Check whether an element is writable."""
         return not ('disabled' in self or 'readonly' in self)
 
-    def is_content_editable(self):
+    def is_content_editable(self) -> bool:
         """Check if an element has a contenteditable attribute.
 
         Args:
@@ -181,7 +173,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
         except KeyError:
             return False
 
-    def _is_editable_object(self):
+    def _is_editable_object(self) -> bool:
         """Check if an object-element is editable."""
         if 'type' not in self:
             log.webelem.debug("<object> without type clicked...")
@@ -197,7 +189,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
             # Image/Audio/...
             return False
 
-    def _is_editable_input(self):
+    def _is_editable_input(self) -> bool:
         """Check if an input-element is editable.
 
         Return:
@@ -214,7 +206,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
             else:
                 return False
 
-    def _is_editable_classes(self):
+    def _is_editable_classes(self) -> bool:
         """Check if an element is editable based on its classes.
 
         Return:
@@ -233,7 +225,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
                 return True
         return False
 
-    def is_editable(self, strict=False):
+    def is_editable(self, strict: bool = False) -> bool:
         """Check whether we should switch to insert mode for this element.
 
         Args:
@@ -264,17 +256,17 @@ class AbstractWebElement(collections.abc.MutableMapping):
             return self._is_editable_classes() and not strict
         return False
 
-    def is_text_input(self):
+    def is_text_input(self) -> bool:
         """Check if this element is some kind of text box."""
         roles = ('combobox', 'textbox')
         tag = self.tag_name()
         return self.get('role', None) in roles or tag in ['input', 'textarea']
 
-    def remove_blank_target(self):
+    def remove_blank_target(self) -> None:
         """Remove target from link."""
         raise NotImplementedError
 
-    def resolve_url(self, baseurl):
+    def resolve_url(self, baseurl: QUrl) -> typing.Optional[QUrl]:
         """Resolve the URL in the element's src/href attribute.
 
         Args:
@@ -301,16 +293,16 @@ class AbstractWebElement(collections.abc.MutableMapping):
         qtutils.ensure_valid(url)
         return url
 
-    def is_link(self):
+    def is_link(self) -> bool:
         """Return True if this AbstractWebElement is a link."""
         href_tags = ['a', 'area', 'link']
         return self.tag_name() in href_tags and 'href' in self
 
-    def _requires_user_interaction(self):
+    def _requires_user_interaction(self) -> bool:
         """Return True if clicking this element needs user interaction."""
         raise NotImplementedError
 
-    def _mouse_pos(self):
+    def _mouse_pos(self) -> QPoint:
         """Get the position to click/hover."""
         # Click the center of the largest square fitting into the top/left
         # corner of the rectangle, this will help if part of the <a> element
@@ -326,35 +318,38 @@ class AbstractWebElement(collections.abc.MutableMapping):
             raise Error("Element position is out of view!")
         return pos
 
-    def _move_text_cursor(self):
+    def _move_text_cursor(self) -> None:
         """Move cursor to end after clicking."""
         raise NotImplementedError
 
-    def _click_fake_event(self, click_target):
+    def _click_fake_event(self, click_target: usertypes.ClickTarget) -> None:
         """Send a fake click event to the element."""
         pos = self._mouse_pos()
 
         log.webelem.debug("Sending fake click to {!r} at position {} with "
                           "target {}".format(self, pos, click_target))
 
-        modifiers = {
+        target_modifiers = {
             usertypes.ClickTarget.normal: Qt.NoModifier,
             usertypes.ClickTarget.window: Qt.AltModifier | Qt.ShiftModifier,
             usertypes.ClickTarget.tab: Qt.ControlModifier,
             usertypes.ClickTarget.tab_bg: Qt.ControlModifier,
         }
         if config.val.tabs.background:
-            modifiers[usertypes.ClickTarget.tab] |= Qt.ShiftModifier
+            target_modifiers[usertypes.ClickTarget.tab] |= Qt.ShiftModifier
         else:
-            modifiers[usertypes.ClickTarget.tab_bg] |= Qt.ShiftModifier
+            target_modifiers[usertypes.ClickTarget.tab_bg] |= Qt.ShiftModifier
+
+        modifiers = typing.cast(Qt.KeyboardModifiers,
+                                target_modifiers[click_target])
 
         events = [
             QMouseEvent(QEvent.MouseMove, pos, Qt.NoButton, Qt.NoButton,
                         Qt.NoModifier),
             QMouseEvent(QEvent.MouseButtonPress, pos, Qt.LeftButton,
-                        Qt.LeftButton, modifiers[click_target]),
+                        Qt.LeftButton, modifiers),
             QMouseEvent(QEvent.MouseButtonRelease, pos, Qt.LeftButton,
-                        Qt.NoButton, modifiers[click_target]),
+                        Qt.NoButton, modifiers),
         ]
 
         for evt in events:
@@ -362,15 +357,15 @@ class AbstractWebElement(collections.abc.MutableMapping):
 
         QTimer.singleShot(0, self._move_text_cursor)
 
-    def _click_editable(self, click_target):
+    def _click_editable(self, click_target: usertypes.ClickTarget) -> None:
         """Fake a click on an editable input field."""
         raise NotImplementedError
 
-    def _click_js(self, click_target):
+    def _click_js(self, click_target: usertypes.ClickTarget) -> None:
         """Fake a click by using the JS .click() method."""
         raise NotImplementedError
 
-    def _click_href(self, click_target):
+    def _click_href(self, click_target: usertypes.ClickTarget) -> None:
         """Fake a click on an element with a href by opening the link."""
         baseurl = self._tab.url()
         url = self.resolve_url(baseurl)
@@ -386,13 +381,14 @@ class AbstractWebElement(collections.abc.MutableMapping):
             background = click_target == usertypes.ClickTarget.tab_bg
             tabbed_browser.tabopen(url, background=background)
         elif click_target == usertypes.ClickTarget.window:
-            window = mainwindow.MainWindow(private=tabbed_browser.private)
+            window = mainwindow.MainWindow(private=tabbed_browser.is_private)
             window.show()
             window.tabbed_browser.tabopen(url)
         else:
             raise ValueError("Unknown ClickTarget {}".format(click_target))
 
-    def click(self, click_target, *, force_event=False):
+    def click(self, click_target: usertypes.ClickTarget, *,
+              force_event: bool = False) -> None:
         """Simulate a click on the element.
 
         Args:
@@ -429,7 +425,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
         else:
             raise ValueError("Unknown ClickTarget {}".format(click_target))
 
-    def hover(self):
+    def hover(self) -> None:
         """Simulate a mouse hover over the element."""
         pos = self._mouse_pos()
         event = QMouseEvent(QEvent.MouseMove, pos, Qt.NoButton, Qt.NoButton,

qutebrowser/browser/webengine/cookies.py

@@ -20,7 +20,7 @@
 """Filter for QtWebEngine cookies."""
 
 from qutebrowser.config import config
-from qutebrowser.utils import utils
+from qutebrowser.utils import utils, qtutils
 
 
 def _accept_cookie(request):
@@ -29,7 +29,13 @@ def _accept_cookie(request):
     if accept == 'all':
         return True
     elif accept in ['no-3rdparty', 'no-unknown-3rdparty']:
-        return not request.thirdParty
+        if qtutils.version_check('5.11.3', compiled=False):
+            third_party = request.thirdParty
+        else:
+            # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-71393
+            third_party = (request.thirdParty and
+                           not request.firstPartyUrl.isEmpty())
+        return not third_party
     elif accept == 'never':
         return False
     else:

qutebrowser/browser/webengine/interceptor.py

@@ -26,15 +26,15 @@ from PyQt5.QtWebEngineCore import (QWebEngineUrlRequestInterceptor,
 from qutebrowser.config import config
 from qutebrowser.browser import shared
 from qutebrowser.utils import utils, log, debug
+from qutebrowser.extensions import interceptors
 
 
 class RequestInterceptor(QWebEngineUrlRequestInterceptor):
 
     """Handle ad blocking and custom headers."""
 
-    def __init__(self, host_blocker, args, parent=None):
+    def __init__(self, args, parent=None):
         super().__init__(parent)
-        self._host_blocker = host_blocker
         self._args = args
 
     def install(self, profile):
@@ -84,9 +84,10 @@ class RequestInterceptor(QWebEngineUrlRequestInterceptor):
                 return
 
         # FIXME:qtwebengine only block ads for NavigationTypeOther?
-        if self._host_blocker.is_blocked(url, first_party):
+        request = interceptors.Request(first_party_url=first_party,
-            log.webview.info("Request to {} blocked by host blocker.".format(
+                                       request_url=url)
-                url.host()))
+        interceptors.run(request)
+        if request.is_blocked:
             info.block(True)
 
         for header, value in shared.custom_headers(url=url):

qutebrowser/browser/webengine/webenginedownloads.py

@@ -117,8 +117,7 @@ class DownloadItem(downloads.AbstractDownloadItem):
     def _get_open_filename(self):
         return self._filename
 
-    def _set_fileobj(self, fileobj, *,
+    def _set_fileobj(self, fileobj, *, autoclose=True):
-                     autoclose=True):  # pylint: disable=unused-argument
         raise downloads.UnsupportedOperationError
 
     def _set_tempfile(self, fileobj):

qutebrowser/browser/webengine/webengineelem.py

@@ -22,20 +22,27 @@
 
 """QtWebEngine specific part of the web element API."""
 
+import typing
+
 from PyQt5.QtCore import QRect, Qt, QPoint, QEventLoop
 from PyQt5.QtGui import QMouseEvent
 from PyQt5.QtWidgets import QApplication
 from PyQt5.QtWebEngineWidgets import QWebEngineSettings
 
-from qutebrowser.utils import log, javascript, urlutils
+from qutebrowser.utils import log, javascript, urlutils, usertypes
 from qutebrowser.browser import webelem
+MYPY = False
+if MYPY:
+    # pylint: disable=unused-import,useless-suppression
+    from qutebrowser.browser.webengine import webenginetab
 
 
 class WebEngineElement(webelem.AbstractWebElement):
 
     """A web element for QtWebEngine, using JS under the hood."""
 
-    def __init__(self, js_dict, tab):
+    def __init__(self, js_dict: typing.Dict[str, typing.Any],
+                 tab: 'webenginetab.WebEngineTab') -> None:
         super().__init__(tab)
         # Do some sanity checks on the data we get from JS
         js_dict_types = {
@@ -48,7 +55,7 @@ class WebEngineElement(webelem.AbstractWebElement):
             'rects': list,
             'attributes': dict,
             'caret_position': (int, type(None)),
-        }
+        }  # type: typing.Dict[str, typing.Union[type, typing.Tuple[type,...]]]
         assert set(js_dict.keys()).issubset(js_dict_types.keys())
         for name, typ in js_dict_types.items():
             if name in js_dict and not isinstance(js_dict[name], typ):
@@ -73,50 +80,51 @@ class WebEngineElement(webelem.AbstractWebElement):
         self._id = js_dict['id']
         self._js_dict = js_dict
 
-    def __str__(self):
+    def __str__(self) -> str:
         return self._js_dict.get('text', '')
 
-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
         if not isinstance(other, WebEngineElement):
             return NotImplemented
         return self._id == other._id  # pylint: disable=protected-access
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> str:
         attrs = self._js_dict['attributes']
         return attrs[key]
 
-    def __setitem__(self, key, val):
+    def __setitem__(self, key: str, val: str) -> None:
         self._js_dict['attributes'][key] = val
         self._js_call('set_attribute', key, val)
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: str) -> None:
         log.stub()
 
-    def __iter__(self):
+    def __iter__(self) -> typing.Iterator[str]:
         return iter(self._js_dict['attributes'])
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self._js_dict['attributes'])
 
-    def _js_call(self, name, *args, callback=None):
+    def _js_call(self, name: str, *args: webelem.JsValueType,
+                 callback: typing.Callable[[typing.Any], None] = None) -> None:
         """Wrapper to run stuff from webelem.js."""
         if self._tab.is_deleted():
             raise webelem.OrphanedError("Tab containing element vanished")
         js_code = javascript.assemble('webelem', name, self._id, *args)
         self._tab.run_js_async(js_code, callback=callback)
 
-    def has_frame(self):
+    def has_frame(self) -> bool:
         return True
 
-    def geometry(self):
+    def geometry(self) -> QRect:
         log.stub()
         return QRect()
 
-    def classes(self):
+    def classes(self) -> typing.List[str]:
         """Get a list of classes assigned to this element."""
         return self._js_dict['class_name'].split()
 
-    def tag_name(self):
+    def tag_name(self) -> str:
         """Get the tag name of this element.
 
         The returned name will always be lower-case.
@@ -125,34 +133,37 @@ class WebEngineElement(webelem.AbstractWebElement):
         assert isinstance(tag, str), tag
         return tag.lower()
 
-    def outer_xml(self):
+    def outer_xml(self) -> str:
         """Get the full HTML representation of this element."""
         return self._js_dict['outer_xml']
 
-    def value(self):
+    def value(self) -> webelem.JsValueType:
         return self._js_dict.get('value', None)
 
-    def set_value(self, value):
+    def set_value(self, value: webelem.JsValueType) -> None:
         self._js_call('set_value', value)
 
-    def dispatch_event(self, event, bubbles=False,
+    def dispatch_event(self, event: str,
-                       cancelable=False, composed=False):
+                       bubbles: bool = False,
+                       cancelable: bool = False,
+                       composed: bool = False) -> None:
         self._js_call('dispatch_event', event, bubbles, cancelable, composed)
 
-    def caret_position(self):
+    def caret_position(self) -> typing.Optional[int]:
         """Get the text caret position for the current element.
 
         If the element is not a text element, None is returned.
         """
         return self._js_dict.get('caret_position', None)
 
-    def insert_text(self, text):
+    def insert_text(self, text: str) -> None:
         if not self.is_editable(strict=True):
             raise webelem.Error("Element is not editable!")
         log.webelem.debug("Inserting text into element {!r}".format(self))
         self._js_call('insert_text', text)
 
-    def rect_on_view(self, *, elem_geometry=None, no_js=False):
+    def rect_on_view(self, *, elem_geometry: QRect = None,
+                     no_js: bool = False) -> QRect:
         """Get the geometry of the element relative to the webview.
 
         Skipping of small rectangles is due to <a> elements containing other
@@ -193,16 +204,16 @@ class WebEngineElement(webelem.AbstractWebElement):
             self, rects))
         return QRect()
 
-    def remove_blank_target(self):
+    def remove_blank_target(self) -> None:
         if self._js_dict['attributes'].get('target') == '_blank':
             self._js_dict['attributes']['target'] = '_top'
         self._js_call('remove_blank_target')
 
-    def _move_text_cursor(self):
+    def _move_text_cursor(self) -> None:
         if self.is_text_input() and self.is_editable():
             self._js_call('move_cursor_to_end')
 
-    def _requires_user_interaction(self):
+    def _requires_user_interaction(self) -> bool:
         baseurl = self._tab.url()
         url = self.resolve_url(baseurl)
         if url is None:
@@ -211,7 +222,7 @@ class WebEngineElement(webelem.AbstractWebElement):
             return False
         return url.scheme() not in urlutils.WEBENGINE_SCHEMES
 
-    def _click_editable(self, click_target):
+    def _click_editable(self, click_target: usertypes.ClickTarget) -> None:
         # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-58515
         ev = QMouseEvent(QMouseEvent.MouseButtonPress, QPoint(0, 0),
                          QPoint(0, 0), QPoint(0, 0), Qt.NoButton, Qt.NoButton,
@@ -221,10 +232,11 @@ class WebEngineElement(webelem.AbstractWebElement):
         self._js_call('focus')
         self._move_text_cursor()
 
-    def _click_js(self, _click_target):
+    def _click_js(self, _click_target: usertypes.ClickTarget) -> None:
         # FIXME:qtwebengine Have a proper API for this
         # pylint: disable=protected-access
         view = self._tab._widget
+        assert view is not None
         # pylint: enable=protected-access
         attribute = QWebEngineSettings.JavascriptCanOpenWindows
         could_open_windows = view.settings().testAttribute(attribute)
@@ -238,8 +250,9 @@ class WebEngineElement(webelem.AbstractWebElement):
         qapp.processEvents(QEventLoop.ExcludeSocketNotifiers |
                            QEventLoop.ExcludeUserInputEvents)
 
-        def reset_setting(_arg):
+        def reset_setting(_arg: typing.Any) -> None:
             """Set the JavascriptCanOpenWindows setting to its old value."""
+            assert view is not None
             try:
                 view.settings().setAttribute(attribute, could_open_windows)
             except RuntimeError:

qutebrowser/browser/webengine/webenginequtescheme.py

@@ -22,6 +22,11 @@
 from PyQt5.QtCore import QBuffer, QIODevice, QUrl
 from PyQt5.QtWebEngineCore import (QWebEngineUrlSchemeHandler,
                                    QWebEngineUrlRequestJob)
+try:
+    from PyQt5.QtWebEngineCore import QWebEngineUrlScheme  # type: ignore
+except ImportError:
+    # Added in Qt 5.12
+    QWebEngineUrlScheme = None
 
 from qutebrowser.browser import qutescheme
 from qutebrowser.utils import log, qtutils
@@ -33,8 +38,12 @@ class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
 
     def install(self, profile):
         """Install the handler for qute:// URLs on the given profile."""
+        if QWebEngineUrlScheme is not None:
+            assert QWebEngineUrlScheme.schemeByName(b'qute') is not None
+
         profile.installUrlSchemeHandler(b'qute', self)
-        if qtutils.version_check('5.11', compiled=False):
+        if (qtutils.version_check('5.11', compiled=False) and
+                not qtutils.version_check('5.12', compiled=False)):
             # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-63378
             profile.installUrlSchemeHandler(b'chrome-error', self)
             profile.installUrlSchemeHandler(b'chrome-extension', self)
@@ -130,3 +139,16 @@ class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
             buf.seek(0)
             buf.close()
             job.reply(mimetype.encode('ascii'), buf)
+
+
+def init():
+    """Register the qute:// scheme.
+
+    Note this needs to be called early, before constructing any QtWebEngine
+    classes.
+    """
+    if QWebEngineUrlScheme is not None:
+        scheme = QWebEngineUrlScheme(b'qute')
+        scheme.setFlags(QWebEngineUrlScheme.LocalScheme |
+                        QWebEngineUrlScheme.LocalAccessAllowed)
+        QWebEngineUrlScheme.registerScheme(scheme)

qutebrowser/browser/webengine/webenginesettings.py

@@ -30,7 +30,7 @@ from PyQt5.QtGui import QFont
 from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile,
                                       QWebEnginePage)
 
-from qutebrowser.browser.webengine import spell
+from qutebrowser.browser.webengine import spell, webenginequtescheme
 from qutebrowser.config import config, websettings
 from qutebrowser.config.websettings import AttributeInfo as Attr
 from qutebrowser.utils import utils, standarddir, qtutils, message, log
@@ -298,6 +298,7 @@ def init(args):
             not hasattr(QWebEnginePage, 'setInspectedPage')):  # only Qt < 5.11
         os.environ['QTWEBENGINE_REMOTE_DEBUGGING'] = str(utils.random_port())
 
+    webenginequtescheme.init()
     spell.init()
 
     _init_profiles()

qutebrowser/browser/webengine/webenginetab.py

@@ -24,9 +24,9 @@ import functools
 import re
 import html as html_utils
 
-from PyQt5.QtCore import (pyqtSignal, pyqtSlot, Qt, QEvent, QPoint, QPointF,
+from PyQt5.QtCore import (pyqtSignal, pyqtSlot, Qt, QPoint, QPointF, QUrl,
-                          QUrl, QTimer, QObject)
+                          QTimer, QObject)
-from PyQt5.QtGui import QKeyEvent, QIcon
+from PyQt5.QtGui import QIcon
 from PyQt5.QtNetwork import QAuthenticator
 from PyQt5.QtWidgets import QApplication
 from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineScript
@@ -60,10 +60,8 @@ def init():
     _qute_scheme_handler.install(webenginesettings.private_profile)
 
     log.init.debug("Initializing request interceptor...")
-    host_blocker = objreg.get('host-blocker')
     args = objreg.get('args')
-    req_interceptor = interceptor.RequestInterceptor(
+    req_interceptor = interceptor.RequestInterceptor(args=args, parent=app)
-        host_blocker, args=args, parent=app)
     req_interceptor.install(webenginesettings.default_profile)
     req_interceptor.install(webenginesettings.private_profile)
 
@@ -132,7 +130,7 @@ class WebEnginePrinting(browsertab.AbstractPrinting):
     """QtWebEngine implementations related to printing."""
 
     def check_pdf_support(self):
-        return True
+        pass
 
     def check_printer_support(self):
         if not hasattr(self._widget.page(), 'print'):
@@ -205,8 +203,8 @@ class WebEngineSearch(browsertab.AbstractSearch):
 
         self._widget.findText(text, flags, wrapped_callback)
 
-    def search(self, text, *, ignore_case='never', reverse=False,
+    def search(self, text, *, ignore_case=usertypes.IgnoreCase.never,
-               result_cb=None):
+               reverse=False, result_cb=None):
         # Don't go to next entry on duplicate search
         if self.text == text and self.search_displayed:
             log.webview.debug("Ignoring duplicate search request"
@@ -423,7 +421,7 @@ class WebEngineScroller(browsertab.AbstractScroller):
     def _repeated_key_press(self, key, count=1, modifier=Qt.NoModifier):
         """Send count fake key presses to this scroller's WebEngineTab."""
         for _ in range(min(count, 1000)):
-            self._tab.key_press(key, modifier)
+            self._tab.fake_key_press(key, modifier)
 
     @pyqtSlot(QPointF)
     def _update_pos(self, pos):
@@ -478,7 +476,7 @@ class WebEngineScroller(browsertab.AbstractScroller):
     def to_anchor(self, name):
         url = self._tab.url()
         url.setFragment(name)
-        self._tab.openurl(url)
+        self._tab.load_url(url)
 
     def delta(self, x=0, y=0):
         self._tab.run_js_async(javascript.assemble('window', 'scrollBy', x, y))
@@ -500,10 +498,10 @@ class WebEngineScroller(browsertab.AbstractScroller):
         self._repeated_key_press(Qt.Key_Right, count)
 
     def top(self):
-        self._tab.key_press(Qt.Key_Home)
+        self._tab.fake_key_press(Qt.Key_Home)
 
     def bottom(self):
-        self._tab.key_press(Qt.Key_End)
+        self._tab.fake_key_press(Qt.Key_End)
 
     def page_up(self, count=1):
         self._repeated_key_press(Qt.Key_PageUp, count)
@@ -518,25 +516,9 @@ class WebEngineScroller(browsertab.AbstractScroller):
         return self._at_bottom
 
 
-class WebEngineHistory(browsertab.AbstractHistory):
+class WebEngineHistoryPrivate(browsertab.AbstractHistoryPrivate):
 
-    """QtWebEngine implementations related to page history."""
+    """History-related methods which are not part of the extension API."""
-
-    def current_idx(self):
-        return self._history.currentItemIndex()
-
-    def can_go_back(self):
-        return self._history.canGoBack()
-
-    def can_go_forward(self):
-        return self._history.canGoForward()
-
-    def _item_at(self, i):
-        return self._history.itemAt(i)
-
-    def _go_to_item(self, item):
-        self._tab.predicted_navigation.emit(item.url())
-        self._history.goToItem(item)
 
     def serialize(self):
         if not qtutils.version_check('5.9', compiled=False):
@@ -551,11 +533,11 @@ class WebEngineHistory(browsertab.AbstractHistory):
         return qtutils.serialize(self._history)
 
     def deserialize(self, data):
-        return qtutils.deserialize(data, self._history)
+        qtutils.deserialize(data, self._history)
 
     def load_items(self, items):
         if items:
-            self._tab.predicted_navigation.emit(items[-1].url)
+            self._tab.before_load_started.emit(items[-1].url)
 
         stream, _data, cur_data = tabhistory.serialize(items)
         qtutils.deserialize_stream(stream, self._history)
@@ -573,6 +555,37 @@ class WebEngineHistory(browsertab.AbstractHistory):
                 self._tab.load_finished.connect(_on_load_finished)
 
 
+class WebEngineHistory(browsertab.AbstractHistory):
+
+    """QtWebEngine implementations related to page history."""
+
+    def __init__(self, tab):
+        super().__init__(tab)
+        self.private_api = WebEngineHistoryPrivate(tab)
+
+    def __len__(self):
+        return len(self._history)
+
+    def __iter__(self):
+        return iter(self._history.items())
+
+    def current_idx(self):
+        return self._history.currentItemIndex()
+
+    def can_go_back(self):
+        return self._history.canGoBack()
+
+    def can_go_forward(self):
+        return self._history.canGoForward()
+
+    def _item_at(self, i):
+        return self._history.itemAt(i)
+
+    def _go_to_item(self, item):
+        self._tab.before_load_started.emit(item.url())
+        self._history.goToItem(item)
+
+
 class WebEngineZoom(browsertab.AbstractZoom):
 
     """QtWebEngine implementations related to zooming."""
@@ -585,19 +598,20 @@ class WebEngineElements(browsertab.AbstractElements):
 
     """QtWebEngine implemementations related to elements on the page."""
 
-    def _js_cb_multiple(self, callback, js_elems):
+    def _js_cb_multiple(self, callback, error_cb, js_elems):
         """Handle found elements coming from JS and call the real callback.
 
         Args:
             callback: The callback to call with the found elements.
-                      Called with None if there was an error.
+            error_cb: The callback to call in case of an error.
             js_elems: The elements serialized from javascript.
         """
         if js_elems is None:
-            callback(None)
+            error_cb(webelem.Error("Unknown error while getting "
+                                   "elements"))
             return
         elif not js_elems['success']:
-            callback(webelem.Error(js_elems['error']))
+            error_cb(webelem.Error(js_elems['error']))
             return
 
         elems = []
@@ -624,10 +638,11 @@ class WebEngineElements(browsertab.AbstractElements):
             elem = webengineelem.WebEngineElement(js_elem, tab=self._tab)
             callback(elem)
 
-    def find_css(self, selector, callback, *, only_visible=False):
+    def find_css(self, selector, callback, error_cb, *,
+                 only_visible=False):
         js_code = javascript.assemble('webelem', 'find_css', selector,
                                       only_visible)
-        js_cb = functools.partial(self._js_cb_multiple, callback)
+        js_cb = functools.partial(self._js_cb_multiple, callback, error_cb)
         self._tab.run_js_async(js_code, js_cb)
 
     def find_id(self, elem_id, callback):
@@ -670,8 +685,9 @@ class WebEngineAudio(browsertab.AbstractAudio):
         self._tab.url_changed.connect(self._on_url_changed)
         config.instance.changed.connect(self._on_config_changed)
 
-    def set_muted(self, muted: bool, override: bool = False):
+    def set_muted(self, muted: bool, override: bool = False) -> None:
         self._overridden = override
+        assert self._widget is not None
         page = self._widget.page()
         page.setAudioMuted(muted)
 
@@ -1031,6 +1047,28 @@ class _WebEngineScripts(QObject):
             page_scripts.insert(new_script)
 
 
+class WebEngineTabPrivate(browsertab.AbstractTabPrivate):
+
+    """QtWebEngine-related methods which aren't part of the public API."""
+
+    def networkaccessmanager(self):
+        return None
+
+    def user_agent(self):
+        return None
+
+    def clear_ssl_errors(self):
+        raise browsertab.UnsupportedOperationError
+
+    def event_target(self):
+        return self._widget.render_widget()
+
+    def shutdown(self):
+        self._tab.shutting_down.emit()
+        self._tab.action.exit_fullscreen()
+        self._widget.shutdown()
+
+
 class WebEngineTab(browsertab.AbstractTab):
 
     """A QtWebEngine tab in the browser.
@@ -1044,8 +1082,7 @@ class WebEngineTab(browsertab.AbstractTab):
     _load_finished_fake = pyqtSignal(bool)
 
     def __init__(self, *, win_id, mode_manager, private, parent=None):
-        super().__init__(win_id=win_id, mode_manager=mode_manager,
+        super().__init__(win_id=win_id, private=private, parent=parent)
-                         private=private, parent=parent)
         widget = webview.WebEngineView(tabdata=self.data, win_id=win_id,
                                        private=private)
         self.history = WebEngineHistory(tab=self)
@@ -1058,6 +1095,8 @@ class WebEngineTab(browsertab.AbstractTab):
         self.elements = WebEngineElements(tab=self)
         self.action = WebEngineAction(tab=self)
         self.audio = WebEngineAudio(tab=self, parent=self)
+        self.private_api = WebEngineTabPrivate(mode_manager=mode_manager,
+                                               tab=self)
         self._permissions = _WebEnginePermissions(tab=self, parent=self)
         self._scripts = _WebEngineScripts(tab=self, parent=self)
         # We're assigning settings in _set_widget
@@ -1095,21 +1134,23 @@ class WebEngineTab(browsertab.AbstractTab):
         self.zoom.set_factor(self._saved_zoom)
         self._saved_zoom = None
 
-    def openurl(self, url, *, predict=True):
+    def load_url(self, url, *, emit_before_load_started=True):
-        """Open the given URL in this tab.
+        """Load the given URL in this tab.
 
         Arguments:
-            url: The QUrl to open.
+            url: The QUrl to load.
-            predict: If set to False, predicted_navigation is not emitted.
+            emit_before_load_started: If set to False, before_load_started is
+                                      not emitted.
         """
         if sip.isdeleted(self._widget):
             # https://github.com/qutebrowser/qutebrowser/issues/3896
             return
         self._saved_zoom = self.zoom.factor()
-        self._openurl_prepare(url, predict=predict)
+        self._load_url_prepare(
+            url, emit_before_load_started=emit_before_load_started)
         self._widget.load(url)
 
-    def url(self, requested=False):
+    def url(self, *, requested=False):
         page = self._widget.page()
         if requested:
             return page.requestedUrl()
@@ -1139,11 +1180,6 @@ class WebEngineTab(browsertab.AbstractTab):
         else:
             self._widget.page().runJavaScript(code, world_id, callback)
 
-    def shutdown(self):
-        self.shutting_down.emit()
-        self.action.exit_fullscreen()
-        self._widget.shutdown()
-
     def reload(self, *, force=False):
         if force:
             action = QWebEnginePage.ReloadAndBypassCache
@@ -1168,22 +1204,6 @@ class WebEngineTab(browsertab.AbstractTab):
         # percent encoded content is 2 megabytes minus 30 bytes.
         self._widget.setHtml(html, base_url)
 
-    def networkaccessmanager(self):
-        return None
-
-    def user_agent(self):
-        return None
-
-    def clear_ssl_errors(self):
-        raise browsertab.UnsupportedOperationError
-
-    def key_press(self, key, modifier=Qt.NoModifier):
-        press_evt = QKeyEvent(QEvent.KeyPress, key, modifier, 0, 0, 0)
-        release_evt = QKeyEvent(QEvent.KeyRelease, key, modifier,
-                                0, 0, 0)
-        self.send_event(press_evt)
-        self.send_event(release_evt)
-
     def _show_error_page(self, url, error):
         """Show an error page in the tab."""
         log.misc.debug("Showing error page for {}".format(error))
@@ -1220,7 +1240,7 @@ class WebEngineTab(browsertab.AbstractTab):
             log.misc.debug("Ignoring invalid URL being added to history")
             return
 
-        self.add_history_item.emit(url, requested_url, title)
+        self.history_item_triggered.emit(url, requested_url, title)
 
     @pyqtSlot(QUrl, 'QAuthenticator*', 'QString')
     def _on_proxy_authentication_required(self, url, authenticator,
@@ -1246,15 +1266,21 @@ class WebEngineTab(browsertab.AbstractTab):
 
     @pyqtSlot(QUrl, 'QAuthenticator*')
     def _on_authentication_required(self, url, authenticator):
+        log.network.debug("Authentication requested for {}, netrc_used {}"
+                          .format(url.toDisplayString(), self.data.netrc_used))
+
         netrc_success = False
         if not self.data.netrc_used:
             self.data.netrc_used = True
             netrc_success = shared.netrc_authentication(url, authenticator)
+
         if not netrc_success:
+            log.network.debug("Asking for credentials")
             abort_on = [self.shutting_down, self.load_started]
             answer = shared.authentication_required(url, authenticator,
                                                     abort_on)
         if not netrc_success and answer is None:
+            log.network.debug("Aborting auth")
             try:
                 # pylint: disable=no-member, useless-suppression
                 sip.assign(authenticator, QAuthenticator())
@@ -1342,9 +1368,9 @@ class WebEngineTab(browsertab.AbstractTab):
             log.config.debug(
                 "Loading {} again because of config change".format(
                     self._reload_url.toDisplayString()))
-            QTimer.singleShot(100, functools.partial(self.openurl,
+            QTimer.singleShot(100, functools.partial(
-                                                     self._reload_url,
+                self.load_url, self._reload_url,
-                                                     predict=False))
+                emit_before_load_started=False))
             self._reload_url = None
 
         if not qtutils.version_check('5.10', compiled=False):
@@ -1383,12 +1409,12 @@ class WebEngineTab(browsertab.AbstractTab):
             self._show_error_page(url, str(error))
 
     @pyqtSlot(QUrl)
-    def _on_predicted_navigation(self, url):
+    def _on_before_load_started(self, url):
-        """If we know we're going to visit an URL soon, change the settings.
+        """If we know we're going to visit a URL soon, change the settings.
 
         This is a WORKAROUND for https://bugreports.qt.io/browse/QTBUG-66656
         """
-        super()._on_predicted_navigation(url)
+        super()._on_before_load_started(url)
         if not qtutils.version_check('5.11.1', compiled=False):
             self.settings.update_for_url(url)
 
@@ -1466,12 +1492,9 @@ class WebEngineTab(browsertab.AbstractTab):
             page.loadFinished.connect(self._restore_zoom)
             page.loadFinished.connect(self._on_load_finished)
 
-        self.predicted_navigation.connect(self._on_predicted_navigation)
+        self.before_load_started.connect(self._on_before_load_started)
 
         # pylint: disable=protected-access
         self.audio._connect_signals()
         self._permissions.connect_signals()
         self._scripts.connect_signals()
-
-    def event_target(self):
-        return self._widget.render_widget()

qutebrowser/browser/webengine/webview.py

@@ -240,7 +240,7 @@ class WebEnginePage(QWebEnginePage):
     def acceptNavigationRequest(self,
                                 url: QUrl,
                                 typ: QWebEnginePage.NavigationType,
-                                is_main_frame: bool):
+                                is_main_frame: bool) -> bool:
         """Override acceptNavigationRequest to forward it to the tab API."""
         type_map = {
             QWebEnginePage.NavigationTypeLinkClicked:

qutebrowser/browser/webkit/mhtml.py

@@ -39,6 +39,7 @@ from PyQt5.QtCore import QUrl
 from qutebrowser.browser import downloads
 from qutebrowser.browser.webkit import webkitelem
 from qutebrowser.utils import log, objreg, message, usertypes, utils, urlutils
+from qutebrowser.extensions import interceptors
 
 
 @attr.s
@@ -354,8 +355,9 @@ class _Downloader:
         # qute, see the comments/discussion on
         # https://github.com/qutebrowser/qutebrowser/pull/962#discussion_r40256987
         # and https://github.com/qutebrowser/qutebrowser/issues/1053
-        host_blocker = objreg.get('host-blocker')
+        request = interceptors.Request(first_party_url=None, request_url=url)
-        if host_blocker.is_blocked(url):
+        interceptors.run(request)
+        if request.is_blocked:
             log.downloads.debug("Skipping {}, host-blocked".format(url))
             # We still need an empty file in the output, QWebView can be pretty
             # picky about displaying a file correctly when not all assets are
@@ -516,7 +518,6 @@ class _NoCloseBytesIO(io.BytesIO):
 
     def close(self):
         """Do nothing."""
-        pass
 
     def actual_close(self):
         """Close the stream."""

qutebrowser/browser/webkit/network/networkmanager.py

@@ -21,6 +21,7 @@
 
 import collections
 import html
+import typing  # pylint: disable=unused-import
 
 import attr
 from PyQt5.QtCore import (pyqtSlot, pyqtSignal, QCoreApplication, QUrl,
@@ -28,16 +29,23 @@ from PyQt5.QtCore import (pyqtSlot, pyqtSignal, QCoreApplication, QUrl,
 from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkReply, QSslSocket
 
 from qutebrowser.config import config
+
+MYPY = False
+if MYPY:
+    # pylint can't interpret type comments with Python 3.7
+    # pylint: disable=unused-import,useless-suppression
+    from qutebrowser.mainwindow import prompt
 from qutebrowser.utils import (message, log, usertypes, utils, objreg,
                                urlutils, debug)
 from qutebrowser.browser import shared
+from qutebrowser.extensions import interceptors
 from qutebrowser.browser.webkit import certificateerror
 from qutebrowser.browser.webkit.network import (webkitqutescheme, networkreply,
                                                 filescheme)
 
 
 HOSTBLOCK_ERROR_STRING = '%HOSTBLOCK%'
-_proxy_auth_cache = {}
+_proxy_auth_cache = {}  # type: typing.Dict[ProxyId, prompt.AuthInfo]
 
 
 @attr.s(frozen=True)
@@ -275,14 +283,19 @@ class NetworkManager(QNetworkAccessManager):
     @pyqtSlot('QNetworkReply*', 'QAuthenticator*')
     def on_authentication_required(self, reply, authenticator):
         """Called when a website needs authentication."""
+        url = reply.url()
+        log.network.debug("Authentication requested for {}, netrc_used {}"
+                          .format(url.toDisplayString(), self.netrc_used))
+
         netrc_success = False
         if not self.netrc_used:
             self.netrc_used = True
-            netrc_success = shared.netrc_authentication(reply.url(),
+            netrc_success = shared.netrc_authentication(url, authenticator)
-                                                        authenticator)
+
         if not netrc_success:
+            log.network.debug("Asking for credentials")
             abort_on = self._get_abort_signals(reply)
-            shared.authentication_required(reply.url(), authenticator,
+            shared.authentication_required(url, authenticator,
                                            abort_on=abort_on)
 
     @pyqtSlot('QNetworkProxy', 'QAuthenticator*')
@@ -290,9 +303,9 @@ class NetworkManager(QNetworkAccessManager):
         """Called when a proxy needs authentication."""
         proxy_id = ProxyId(proxy.type(), proxy.hostName(), proxy.port())
         if proxy_id in _proxy_auth_cache:
-            user, password = _proxy_auth_cache[proxy_id]
+            authinfo = _proxy_auth_cache[proxy_id]
-            authenticator.setUser(user)
+            authenticator.setUser(authinfo.user)
-            authenticator.setPassword(password)
+            authenticator.setPassword(authinfo.password)
         else:
             msg = '<b>{}</b> says:<br/>{}'.format(
                 html.escape(proxy.hostName()),
@@ -393,10 +406,10 @@ class NetworkManager(QNetworkAccessManager):
                 # the webpage shutdown here.
                 current_url = QUrl()
 
-        host_blocker = objreg.get('host-blocker')
+        request = interceptors.Request(first_party_url=current_url,
-        if host_blocker.is_blocked(req.url(), current_url):
+                                       request_url=req.url())
-            log.webview.info("Request to {} blocked by host blocker.".format(
+        interceptors.run(request)
-                req.url().host()))
+        if request.is_blocked:
             return networkreply.ErrorNetworkReply(
                 req, HOSTBLOCK_ERROR_STRING, QNetworkReply.ContentAccessDenied,
                 self)

qutebrowser/browser/webkit/network/networkreply.py

@@ -67,7 +67,6 @@ class FixedDataNetworkReply(QNetworkReply):
     @pyqtSlot()
     def abort(self):
         """Abort the operation."""
-        pass
 
     def bytesAvailable(self):
         """Determine the bytes available for being read.
@@ -123,7 +122,6 @@ class ErrorNetworkReply(QNetworkReply):
 
     def abort(self):
         """Do nothing since it's a fake reply."""
-        pass
 
     def bytesAvailable(self):
         """We always have 0 bytes available."""
@@ -151,7 +149,6 @@ class RedirectNetworkReply(QNetworkReply):
 
     def abort(self):
         """Called when there's e.g. a redirection limit."""
-        pass
 
     def readData(self, _maxlen):
         return bytes()

qutebrowser/browser/webkit/webkitelem.py

@@ -19,26 +19,31 @@
 
 """QtWebKit specific part of the web element API."""
 
+import typing
+
 from PyQt5.QtCore import QRect
 from PyQt5.QtWebKit import QWebElement, QWebSettings
+from PyQt5.QtWebKitWidgets import QWebFrame
 
 from qutebrowser.config import config
-from qutebrowser.utils import log, utils, javascript
+from qutebrowser.utils import log, utils, javascript, usertypes
 from qutebrowser.browser import webelem
+MYPY = False
+if MYPY:
+    # pylint: disable=unused-import,useless-suppression
+    from qutebrowser.browser.webkit import webkittab
 
 
 class IsNullError(webelem.Error):
 
     """Gets raised by WebKitElement if an element is null."""
 
-    pass
-
 
 class WebKitElement(webelem.AbstractWebElement):
 
     """A wrapper around a QWebElement."""
 
-    def __init__(self, elem, tab):
+    def __init__(self, elem: QWebElement, tab: 'webkittab.WebKitTab') -> None:
         super().__init__(tab)
         if isinstance(elem, self.__class__):
             raise TypeError("Trying to wrap a wrapper!")
@@ -46,90 +51,94 @@ class WebKitElement(webelem.AbstractWebElement):
             raise IsNullError('{} is a null element!'.format(elem))
         self._elem = elem
 
-    def __str__(self):
+    def __str__(self) -> str:
         self._check_vanished()
         return self._elem.toPlainText()
 
-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
         if not isinstance(other, WebKitElement):
             return NotImplemented
         return self._elem == other._elem  # pylint: disable=protected-access
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> str:
         self._check_vanished()
         if key not in self:
             raise KeyError(key)
         return self._elem.attribute(key)
 
-    def __setitem__(self, key, val):
+    def __setitem__(self, key: str, val: str) -> None:
         self._check_vanished()
         self._elem.setAttribute(key, val)
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: str) -> None:
         self._check_vanished()
         if key not in self:
             raise KeyError(key)
         self._elem.removeAttribute(key)
 
-    def __contains__(self, key):
+    def __contains__(self, key: object) -> bool:
+        assert isinstance(key, str)
         self._check_vanished()
         return self._elem.hasAttribute(key)
 
-    def __iter__(self):
+    def __iter__(self) -> typing.Iterator[str]:
         self._check_vanished()
         yield from self._elem.attributeNames()
 
-    def __len__(self):
+    def __len__(self) -> int:
         self._check_vanished()
         return len(self._elem.attributeNames())
 
-    def _check_vanished(self):
+    def _check_vanished(self) -> None:
         """Raise an exception if the element vanished (is null)."""
         if self._elem.isNull():
             raise IsNullError('Element {} vanished!'.format(self._elem))
 
-    def has_frame(self):
+    def has_frame(self) -> bool:
         self._check_vanished()
         return self._elem.webFrame() is not None
 
-    def geometry(self):
+    def geometry(self) -> QRect:
         self._check_vanished()
         return self._elem.geometry()
 
-    def classes(self):
+    def classes(self) -> typing.List[str]:
         self._check_vanished()
         return self._elem.classes()
 
-    def tag_name(self):
+    def tag_name(self) -> str:
         """Get the tag name for the current element."""
         self._check_vanished()
         return self._elem.tagName().lower()
 
-    def outer_xml(self):
+    def outer_xml(self) -> str:
         """Get the full HTML representation of this element."""
         self._check_vanished()
         return self._elem.toOuterXml()
 
-    def value(self):
+    def value(self) -> webelem.JsValueType:
         self._check_vanished()
         val = self._elem.evaluateJavaScript('this.value')
         assert isinstance(val, (int, float, str, type(None))), val
         return val
 
-    def set_value(self, value):
+    def set_value(self, value: webelem.JsValueType) -> None:
         self._check_vanished()
         if self._tab.is_deleted():
             raise webelem.OrphanedError("Tab containing element vanished")
         if self.is_content_editable():
             log.webelem.debug("Filling {!r} via set_text.".format(self))
+            assert isinstance(value, str)
             self._elem.setPlainText(value)
         else:
             log.webelem.debug("Filling {!r} via javascript.".format(self))
             value = javascript.to_js(value)
             self._elem.evaluateJavaScript("this.value={}".format(value))
 
-    def dispatch_event(self, event, bubbles=False,
+    def dispatch_event(self, event: str,
-                       cancelable=False, composed=False):
+                       bubbles: bool = False,
+                       cancelable: bool = False,
+                       composed: bool = False) -> None:
         self._check_vanished()
         log.webelem.debug("Firing event on {!r} via javascript.".format(self))
         self._elem.evaluateJavaScript(
@@ -140,7 +149,7 @@ class WebKitElement(webelem.AbstractWebElement):
                     javascript.to_js(cancelable),
                     javascript.to_js(composed)))
 
-    def caret_position(self):
+    def caret_position(self) -> int:
         """Get the text caret position for the current element."""
         self._check_vanished()
         pos = self._elem.evaluateJavaScript('this.selectionStart')
@@ -148,7 +157,7 @@ class WebKitElement(webelem.AbstractWebElement):
             return 0
         return int(pos)
 
-    def insert_text(self, text):
+    def insert_text(self, text: str) -> None:
         self._check_vanished()
         if not self.is_editable(strict=True):
             raise webelem.Error("Element is not editable!")
@@ -160,7 +169,7 @@ class WebKitElement(webelem.AbstractWebElement):
             this.dispatchEvent(event);
         """.format(javascript.to_js(text)))
 
-    def _parent(self):
+    def _parent(self) -> typing.Optional['WebKitElement']:
         """Get the parent element of this element."""
         self._check_vanished()
         elem = self._elem.parent()
@@ -168,7 +177,7 @@ class WebKitElement(webelem.AbstractWebElement):
             return None
         return WebKitElement(elem, tab=self._tab)
 
-    def _rect_on_view_js(self):
+    def _rect_on_view_js(self) -> typing.Optional[QRect]:
         """Javascript implementation for rect_on_view."""
         # FIXME:qtwebengine maybe we can reuse this?
         rects = self._elem.evaluateJavaScript("this.getClientRects()")
@@ -180,8 +189,8 @@ class WebKitElement(webelem.AbstractWebElement):
             return None
 
         text = utils.compact_text(self._elem.toOuterXml(), 500)
-        log.webelem.vdebug("Client rectangles of element '{}': {}".format(
+        log.webelem.vdebug(  # type: ignore
-            text, rects))
+            "Client rectangles of element '{}': {}".format(text, rects))
 
         for i in range(int(rects.get("length", 0))):
             rect = rects[str(i)]
@@ -206,7 +215,8 @@ class WebKitElement(webelem.AbstractWebElement):
 
         return None
 
-    def _rect_on_view_python(self, elem_geometry):
+    def _rect_on_view_python(self,
+                             elem_geometry: typing.Optional[QRect]) -> QRect:
         """Python implementation for rect_on_view."""
         if elem_geometry is None:
             geometry = self._elem.geometry()
@@ -220,7 +230,8 @@ class WebKitElement(webelem.AbstractWebElement):
             frame = frame.parentFrame()
         return rect
 
-    def rect_on_view(self, *, elem_geometry=None, no_js=False):
+    def rect_on_view(self, *, elem_geometry: QRect = None,
+                     no_js: bool = False) -> QRect:
         """Get the geometry of the element relative to the webview.
 
         Uses the getClientRects() JavaScript method to obtain the collection of
@@ -250,7 +261,7 @@ class WebKitElement(webelem.AbstractWebElement):
         # No suitable rects found via JS, try via the QWebElement API
         return self._rect_on_view_python(elem_geometry)
 
-    def _is_visible(self, mainframe):
+    def _is_visible(self, mainframe: QWebFrame) -> bool:
         """Check if the given element is visible in the given frame.
 
         This is not public API because it can't be implemented easily here with
@@ -302,8 +313,8 @@ class WebKitElement(webelem.AbstractWebElement):
             visible_in_frame = visible_on_screen
         return all([visible_on_screen, visible_in_frame])
 
-    def remove_blank_target(self):
+    def remove_blank_target(self) -> None:
-        elem = self
+        elem = self  # type: typing.Optional[WebKitElement]
         for _ in range(5):
             if elem is None:
                 break
@@ -313,14 +324,14 @@ class WebKitElement(webelem.AbstractWebElement):
                 break
             elem = elem._parent()  # pylint: disable=protected-access
 
-    def _move_text_cursor(self):
+    def _move_text_cursor(self) -> None:
         if self.is_text_input() and self.is_editable():
             self._tab.caret.move_to_end_of_document()
 
-    def _requires_user_interaction(self):
+    def _requires_user_interaction(self) -> bool:
         return False
 
-    def _click_editable(self, click_target):
+    def _click_editable(self, click_target: usertypes.ClickTarget) -> None:
         ok = self._elem.evaluateJavaScript('this.focus(); true;')
         if ok:
             self._move_text_cursor()
@@ -328,7 +339,7 @@ class WebKitElement(webelem.AbstractWebElement):
             log.webelem.debug("Failed to focus via JS, falling back to event")
             self._click_fake_event(click_target)
 
-    def _click_js(self, click_target):
+    def _click_js(self, click_target: usertypes.ClickTarget) -> None:
         settings = QWebSettings.globalSettings()
         attribute = QWebSettings.JavascriptCanOpenWindows
         could_open_windows = settings.testAttribute(attribute)
@@ -339,12 +350,12 @@ class WebKitElement(webelem.AbstractWebElement):
             log.webelem.debug("Failed to click via JS, falling back to event")
             self._click_fake_event(click_target)
 
-    def _click_fake_event(self, click_target):
+    def _click_fake_event(self, click_target: usertypes.ClickTarget) -> None:
         self._tab.data.override_target = click_target
         super()._click_fake_event(click_target)
 
 
-def get_child_frames(startframe):
+def get_child_frames(startframe: QWebFrame) -> typing.List[QWebFrame]:
     """Get all children recursively of a given QWebFrame.
 
     Loosely based on http://blog.nextgenetics.net/?e=64
@@ -358,7 +369,7 @@ def get_child_frames(startframe):
     results = []
     frames = [startframe]
     while frames:
-        new_frames = []
+        new_frames = []  # type: typing.List[QWebFrame]
         for frame in frames:
             results.append(frame)
             new_frames += frame.childFrames()

qutebrowser/browser/webkit/webkithistory.py

@@ -41,7 +41,6 @@ class WebHistoryInterface(QWebHistoryInterface):
 
     def addHistoryEntry(self, url_string):
         """Required for a QWebHistoryInterface impl, obsoleted by add_url."""
-        pass
 
     @functools.lru_cache(maxsize=32768)
     def historyContains(self, url_string):

qutebrowser/browser/webkit/webkittab.py

@@ -23,9 +23,8 @@ import re
 import functools
 import xml.etree.ElementTree
 
-from PyQt5.QtCore import (pyqtSlot, Qt, QEvent, QUrl, QPoint, QTimer, QSizeF,
+from PyQt5.QtCore import pyqtSlot, Qt, QUrl, QPoint, QTimer, QSizeF, QSize
-                          QSize)
+from PyQt5.QtGui import QIcon
-from PyQt5.QtGui import QKeyEvent, QIcon
 from PyQt5.QtWebKitWidgets import QWebPage, QWebFrame
 from PyQt5.QtWebKit import QWebSettings
 from PyQt5.QtPrintSupport import QPrinter
@@ -125,8 +124,8 @@ class WebKitSearch(browsertab.AbstractSearch):
         self._widget.findText('')
         self._widget.findText('', QWebPage.HighlightAllOccurrences)
 
-    def search(self, text, *, ignore_case='never', reverse=False,
+    def search(self, text, *, ignore_case=usertypes.IgnoreCase.never,
-               result_cb=None):
+               reverse=False, result_cb=None):
         # Don't go to next entry on duplicate search
         if self.text == text and self.search_displayed:
             log.webview.debug("Ignoring duplicate search request"
@@ -391,7 +390,7 @@ class WebKitCaret(browsertab.AbstractCaret):
                 if tab:
                     self._tab.new_tab_requested.emit(url)
                 else:
-                    self._tab.openurl(url)
+                    self._tab.load_url(url)
 
     def follow_selected(self, *, tab=False):
         try:
@@ -474,7 +473,7 @@ class WebKitScroller(browsertab.AbstractScroller):
             if (getter is not None and
                     frame.scrollBarValue(direction) == getter(direction)):
                 return
-            self._tab.key_press(key)
+            self._tab.fake_key_press(key)
 
     def up(self, count=1):
         self._key_press(Qt.Key_Up, count, 'scrollBarMinimum', Qt.Vertical)
@@ -509,35 +508,19 @@ class WebKitScroller(browsertab.AbstractScroller):
         return self.pos_px().y() >= frame.scrollBarMaximum(Qt.Vertical)
 
 
-class WebKitHistory(browsertab.AbstractHistory):
+class WebKitHistoryPrivate(browsertab.AbstractHistoryPrivate):
 
-    """QtWebKit implementations related to page history."""
+    """History-related methods which are not part of the extension API."""
-
-    def current_idx(self):
-        return self._history.currentItemIndex()
-
-    def can_go_back(self):
-        return self._history.canGoBack()
-
-    def can_go_forward(self):
-        return self._history.canGoForward()
-
-    def _item_at(self, i):
-        return self._history.itemAt(i)
-
-    def _go_to_item(self, item):
-        self._tab.predicted_navigation.emit(item.url())
-        self._history.goToItem(item)
 
     def serialize(self):
         return qtutils.serialize(self._history)
 
     def deserialize(self, data):
-        return qtutils.deserialize(data, self._history)
+        qtutils.deserialize(data, self._history)
 
     def load_items(self, items):
         if items:
-            self._tab.predicted_navigation.emit(items[-1].url)
+            self._tab.before_load_started.emit(items[-1].url)
 
         stream, _data, user_data = tabhistory.serialize(items)
         qtutils.deserialize_stream(stream, self._history)
@@ -553,11 +536,43 @@ class WebKitHistory(browsertab.AbstractHistory):
                     self._tab.scroller.to_point, cur_data['scroll-pos']))
 
 
+class WebKitHistory(browsertab.AbstractHistory):
+
+    """QtWebKit implementations related to page history."""
+
+    def __init__(self, tab):
+        super().__init__(tab)
+        self.private_api = WebKitHistoryPrivate(tab)
+
+    def __len__(self):
+        return len(self._history)
+
+    def __iter__(self):
+        return iter(self._history.items())
+
+    def current_idx(self):
+        return self._history.currentItemIndex()
+
+    def can_go_back(self):
+        return self._history.canGoBack()
+
+    def can_go_forward(self):
+        return self._history.canGoForward()
+
+    def _item_at(self, i):
+        return self._history.itemAt(i)
+
+    def _go_to_item(self, item):
+        self._tab.before_load_started.emit(item.url())
+        self._history.goToItem(item)
+
+
 class WebKitElements(browsertab.AbstractElements):
 
     """QtWebKit implemementations related to elements on the page."""
 
-    def find_css(self, selector, callback, *, only_visible=False):
+    def find_css(self, selector, callback, error_cb, *, only_visible=False):
+        utils.unused(error_cb)
         mainframe = self._widget.page().mainFrame()
         if mainframe is None:
             raise browsertab.WebTabError("No frame focused!")
@@ -586,7 +601,7 @@ class WebKitElements(browsertab.AbstractElements):
         # Escape non-alphanumeric characters in the selector
         # https://www.w3.org/TR/CSS2/syndata.html#value-def-identifier
         elem_id = re.sub(r'[^a-zA-Z0-9_-]', r'\\\g<0>', elem_id)
-        self.find_css('#' + elem_id, find_id_cb)
+        self.find_css('#' + elem_id, find_id_cb, error_cb=lambda exc: None)
 
     def find_focused(self, callback):
         frame = self._widget.page().currentFrame()
@@ -641,7 +656,7 @@ class WebKitAudio(browsertab.AbstractAudio):
 
     """Dummy handling of audio status for QtWebKit."""
 
-    def set_muted(self, muted: bool, override: bool = False):
+    def set_muted(self, muted: bool, override: bool = False) -> None:
         raise browsertab.WebTabError('Muting is not supported on QtWebKit!')
 
     def is_muted(self):
@@ -651,13 +666,33 @@ class WebKitAudio(browsertab.AbstractAudio):
         return False
 
 
+class WebKitTabPrivate(browsertab.AbstractTabPrivate):
+
+    """QtWebKit-related methods which aren't part of the public API."""
+
+    def networkaccessmanager(self):
+        return self._widget.page().networkAccessManager()
+
+    def user_agent(self):
+        page = self._widget.page()
+        return page.userAgentForUrl(self._tab.url())
+
+    def clear_ssl_errors(self):
+        self.networkaccessmanager().clear_all_ssl_errors()
+
+    def event_target(self):
+        return self._widget
+
+    def shutdown(self):
+        self._widget.shutdown()
+
+
 class WebKitTab(browsertab.AbstractTab):
 
     """A QtWebKit tab in the browser."""
 
     def __init__(self, *, win_id, mode_manager, private, parent=None):
-        super().__init__(win_id=win_id, mode_manager=mode_manager,
+        super().__init__(win_id=win_id, private=private, parent=parent)
-                         private=private, parent=parent)
         widget = webview.WebView(win_id=win_id, tab_id=self.tab_id,
                                  private=private, tab=self)
         if private:
@@ -672,6 +707,8 @@ class WebKitTab(browsertab.AbstractTab):
         self.elements = WebKitElements(tab=self)
         self.action = WebKitAction(tab=self)
         self.audio = WebKitAudio(tab=self, parent=self)
+        self.private_api = WebKitTabPrivate(mode_manager=mode_manager,
+                                            tab=self)
         # We're assigning settings in _set_widget
         self.settings = webkitsettings.WebKitSettings(settings=None)
         self._set_widget(widget)
@@ -685,11 +722,12 @@ class WebKitTab(browsertab.AbstractTab):
         settings = widget.settings()
         settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
 
-    def openurl(self, url, *, predict=True):
+    def load_url(self, url, *, emit_before_load_started=True):
-        self._openurl_prepare(url, predict=predict)
+        self._load_url_prepare(
-        self._widget.openurl(url)
+            url, emit_before_load_started=emit_before_load_started)
+        self._widget.load(url)
 
-    def url(self, requested=False):
+    def url(self, *, requested=False):
         frame = self._widget.page().mainFrame()
         if requested:
             return frame.requestedUrl()
@@ -714,9 +752,6 @@ class WebKitTab(browsertab.AbstractTab):
     def icon(self):
         return self._widget.icon()
 
-    def shutdown(self):
-        self._widget.shutdown()
-
     def reload(self, *, force=False):
         if force:
             action = QWebPage.ReloadAndBypassCache
@@ -730,36 +765,20 @@ class WebKitTab(browsertab.AbstractTab):
     def title(self):
         return self._widget.title()
 
-    def clear_ssl_errors(self):
-        self.networkaccessmanager().clear_all_ssl_errors()
-
-    def key_press(self, key, modifier=Qt.NoModifier):
-        press_evt = QKeyEvent(QEvent.KeyPress, key, modifier, 0, 0, 0)
-        release_evt = QKeyEvent(QEvent.KeyRelease, key, modifier,
-                                0, 0, 0)
-        self.send_event(press_evt)
-        self.send_event(release_evt)
-
     @pyqtSlot()
     def _on_history_trigger(self):
         url = self.url()
         requested_url = self.url(requested=True)
-        self.add_history_item.emit(url, requested_url, self.title())
+        self.history_item_triggered.emit(url, requested_url, self.title())
 
     def set_html(self, html, base_url=QUrl()):
         self._widget.setHtml(html, base_url)
 
-    def networkaccessmanager(self):
-        return self._widget.page().networkAccessManager()
-
-    def user_agent(self):
-        page = self._widget.page()
-        return page.userAgentForUrl(self.url())
-
     @pyqtSlot()
     def _on_load_started(self):
         super()._on_load_started()
-        self.networkaccessmanager().netrc_used = False
+        nam = self._widget.page().networkAccessManager()
+        nam.netrc_used = False
         # Make sure the icon is cleared when navigating to a page without one.
         self.icon_changed.emit(QIcon())
 
@@ -811,7 +830,7 @@ class WebKitTab(browsertab.AbstractTab):
         if (navigation.navigation_type == navigation.Type.link_clicked and
                 target != usertypes.ClickTarget.normal):
             tab = shared.get_tab(self.win_id, target)
-            tab.openurl(navigation.url)
+            tab.load_url(navigation.url)
             self.data.open_target = usertypes.ClickTarget.normal
             navigation.accepted = False
 
@@ -841,6 +860,3 @@ class WebKitTab(browsertab.AbstractTab):
         frame.contentsSizeChanged.connect(self._on_contents_size_changed)
         frame.initialLayoutCompleted.connect(self._on_history_trigger)
         page.navigation_request.connect(self._on_navigation_request)
-
-    def event_target(self):
-        return self._widget

qutebrowser/browser/webkit/webpage.py

@@ -469,7 +469,7 @@ class BrowserPage(QWebPage):
     def acceptNavigationRequest(self,
                                 frame: QWebFrame,
                                 request: QNetworkRequest,
-                                typ: QWebPage.NavigationType):
+                                typ: QWebPage.NavigationType) -> bool:
         """Override acceptNavigationRequest to handle clicked links.
 
         Setting linkDelegationPolicy to DelegateAllLinks and using a slot bound

qutebrowser/browser/webkit/webview.py

@@ -118,14 +118,6 @@ class WebView(QWebView):
         self.stop()
         self.page().shutdown()
 
-    def openurl(self, url):
-        """Open a URL in the browser.
-
-        Args:
-            url: The URL to load as QUrl
-        """
-        self.load(url)
-
     def createWindow(self, wintype):
         """Called by Qt when a page wants to create a new window.
 

qutebrowser/commands/cmdexc.py

@@ -28,26 +28,15 @@ class Error(Exception):
     """Base class for all cmdexc errors."""
 
 
-class CommandError(Error):
-
-    """Raised when a command encounters an error while running."""
-
-    pass
-
-
 class NoSuchCommandError(Error):
 
     """Raised when a command wasn't found."""
 
-    pass
-
 
 class ArgumentTypeError(Error):
 
     """Raised when an argument had an invalid type."""
 
-    pass
-
 
 class PrerequisitesError(Error):
 
@@ -56,5 +45,3 @@ class PrerequisitesError(Error):
     This is raised for example when we're in the wrong mode while executing the
     command, or we need javascript enabled but don't have done so.
     """
-
-    pass

qutebrowser/commands/cmdutils.py

@@ -1,147 +0,0 @@
-# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
-
-# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Contains various command utils and a global command dict.
-
-Module attributes:
-    cmd_dict: A mapping from command-strings to command objects.
-"""
-
-import inspect
-
-from qutebrowser.utils import qtutils, log
-from qutebrowser.commands import command, cmdexc
-
-cmd_dict = {}
-
-
-def check_overflow(arg, ctype):
-    """Check if the given argument is in bounds for the given type.
-
-    Args:
-        arg: The argument to check
-        ctype: The C/Qt type to check as a string.
-    """
-    try:
-        qtutils.check_overflow(arg, ctype)
-    except OverflowError:
-        raise cmdexc.CommandError(
-            "Numeric argument is too large for internal {} "
-            "representation.".format(ctype))
-
-
-def check_exclusive(flags, names):
-    """Check if only one flag is set with exclusive flags.
-
-    Raise a CommandError if not.
-
-    Args:
-        flags: An iterable of booleans to check.
-        names: An iterable of flag names for the error message.
-    """
-    if sum(1 for e in flags if e) > 1:
-        argstr = '/'.join('-' + e for e in names)
-        raise cmdexc.CommandError("Only one of {} can be given!".format(
-            argstr))
-
-
-class register:  # noqa: N801,N806 pylint: disable=invalid-name
-
-    """Decorator to register a new command handler.
-
-    This could also be a function, but as a class (with a "wrong" name) it's
-    much cleaner to implement.
-
-    Attributes:
-        _instance: The object from the object registry to be used as "self".
-        _name: The name (as string) or names (as list) of the command.
-        _kwargs: The arguments to pass to Command.
-    """
-
-    def __init__(self, *, instance=None, name=None, **kwargs):
-        """Save decorator arguments.
-
-        Gets called on parse-time with the decorator arguments.
-
-        Args:
-            See class attributes.
-        """
-        self._instance = instance
-        self._name = name
-        self._kwargs = kwargs
-
-    def __call__(self, func):
-        """Register the command before running the function.
-
-        Gets called when a function should be decorated.
-
-        Doesn't actually decorate anything, but creates a Command object and
-        registers it in the cmd_dict.
-
-        Args:
-            func: The function to be decorated.
-
-        Return:
-            The original function (unmodified).
-        """
-        if self._name is None:
-            name = func.__name__.lower().replace('_', '-')
-        else:
-            assert isinstance(self._name, str), self._name
-            name = self._name
-        log.commands.vdebug("Registering command {} (from {}:{})".format(
-            name, func.__module__, func.__qualname__))
-        if name in cmd_dict:
-            raise ValueError("{} is already registered!".format(name))
-        cmd = command.Command(name=name, instance=self._instance,
-                              handler=func, **self._kwargs)
-        cmd_dict[name] = cmd
-        return func
-
-
-class argument:  # noqa: N801,N806 pylint: disable=invalid-name
-
-    """Decorator to customize an argument for @cmdutils.register.
-
-    This could also be a function, but as a class (with a "wrong" name) it's
-    much cleaner to implement.
-
-    Attributes:
-        _argname: The name of the argument to handle.
-        _kwargs: Keyword arguments, valid ArgInfo members
-    """
-
-    def __init__(self, argname, **kwargs):
-        self._argname = argname
-        self._kwargs = kwargs
-
-    def __call__(self, func):
-        funcname = func.__name__
-
-        if self._argname not in inspect.signature(func).parameters:
-            raise ValueError("{} has no argument {}!".format(funcname,
-                                                             self._argname))
-        if not hasattr(func, 'qute_args'):
-            func.qute_args = {}
-        elif func.qute_args is None:
-            raise ValueError("@cmdutils.argument got called above (after) "
-                             "@cmdutils.register for {}!".format(funcname))
-
-        func.qute_args[self._argname] = command.ArgInfo(**self._kwargs)
-        return func

qutebrowser/commands/command.py

@@ -26,8 +26,9 @@ import typing
 
 import attr
 
+from qutebrowser.api import cmdutils
 from qutebrowser.commands import cmdexc, argparser
-from qutebrowser.utils import log, message, docutils, objreg, usertypes
+from qutebrowser.utils import log, message, docutils, objreg, usertypes, utils
 from qutebrowser.utils import debug as debug_utils
 from qutebrowser.misc import objects
 
@@ -37,18 +38,13 @@ class ArgInfo:
 
     """Information about an argument."""
 
-    win_id = attr.ib(False)
+    value = attr.ib(None)
-    count = attr.ib(False)
     hide = attr.ib(False)
     metavar = attr.ib(None)
     flag = attr.ib(None)
     completion = attr.ib(None)
     choices = attr.ib(None)
 
-    def __attrs_post_init__(self):
-        if self.win_id and self.count:
-            raise TypeError("Argument marked as both count/win_id!")
-
 
 class Command:
 
@@ -75,6 +71,10 @@ class Command:
         _scope: The scope to get _instance for in the object registry.
     """
 
+    # CommandValue values which need a count
+    COUNT_COMMAND_VALUES = [usertypes.CommandValue.count,
+                            usertypes.CommandValue.count_tab]
+
     def __init__(self, *, handler, name, instance=None, maxsplit=None,
                  modes=None, not_modes=None, debug=False, deprecated=False,
                  no_cmd_split=False, star_args_optional=False, scope='global',
@@ -116,7 +116,6 @@ class Command:
         self.parser.add_argument('-h', '--help', action=argparser.HelpAction,
                                  default=argparser.SUPPRESS, nargs=0,
                                  help=argparser.SUPPRESS)
-        self._check_func()
         self.opt_args = collections.OrderedDict()
         self.namespace = None
         self._count = None
@@ -130,6 +129,7 @@ class Command:
         self._qute_args = getattr(self.handler, 'qute_args', {})
         self.handler.qute_args = None
 
+        self._check_func()
         self._inspect_func()
 
     def _check_prerequisites(self, win_id):
@@ -154,16 +154,21 @@ class Command:
     def _check_func(self):
         """Make sure the function parameters don't violate any rules."""
         signature = inspect.signature(self.handler)
-        if 'self' in signature.parameters and self._instance is None:
+        if 'self' in signature.parameters:
-            raise TypeError("{} is a class method, but instance was not "
+            if self._instance is None:
-                            "given!".format(self.name[0]))
+                raise TypeError("{} is a class method, but instance was not "
+                                "given!".format(self.name))
+            arg_info = self.get_arg_info(signature.parameters['self'])
+            if arg_info.value is not None:
+                raise TypeError("{}: Can't fill 'self' with value!"
+                                .format(self.name))
         elif 'self' not in signature.parameters and self._instance is not None:
             raise TypeError("{} is not a class method, but instance was "
-                            "given!".format(self.name[0]))
+                            "given!".format(self.name))
         elif any(param.kind == inspect.Parameter.VAR_KEYWORD
                  for param in signature.parameters.values()):
             raise TypeError("{}: functions with varkw arguments are not "
-                            "supported!".format(self.name[0]))
+                            "supported!".format(self.name))
 
     def get_arg_info(self, param):
         """Get an ArgInfo tuple for the given inspect.Parameter."""
@@ -186,13 +191,18 @@ class Command:
             True if the parameter is special, False otherwise.
         """
         arg_info = self.get_arg_info(param)
-        if arg_info.count:
+        if arg_info.value is None:
+            return False
+        elif arg_info.value == usertypes.CommandValue.count:
             if param.default is inspect.Parameter.empty:
                 raise TypeError("{}: handler has count parameter "
                                 "without default!".format(self.name))
             return True
-        elif arg_info.win_id:
+        elif isinstance(arg_info.value, usertypes.CommandValue):
             return True
+        else:
+            raise TypeError("{}: Invalid value={!r} for argument '{}'!"
+                            .format(self.name, arg_info.value, param.name))
         return False
 
     def _inspect_func(self):
@@ -292,6 +302,8 @@ class Command:
         name = argparser.arg_name(param.name)
         arg_info = self.get_arg_info(param)
 
+        assert not arg_info.value, name
+
         if arg_info.flag is not None:
             shortname = arg_info.flag
         else:
@@ -321,75 +333,66 @@ class Command:
             param: The inspect.Parameter to look at.
         """
         arginfo = self.get_arg_info(param)
-        if param.annotation is not inspect.Parameter.empty:
+        if arginfo.value:
+            # Filled values are passed 1:1
+            return None
+        elif param.kind in [inspect.Parameter.VAR_POSITIONAL,
+                            inspect.Parameter.VAR_KEYWORD]:
+            # For *args/**kwargs we only support strings
+            assert param.annotation in [inspect.Parameter.empty, str], param
+            return None
+        elif param.annotation is not inspect.Parameter.empty:
             return param.annotation
         elif param.default not in [None, inspect.Parameter.empty]:
             return type(param.default)
-        elif arginfo.count or arginfo.win_id or param.kind in [
-                inspect.Parameter.VAR_POSITIONAL,
-                inspect.Parameter.VAR_KEYWORD]:
-            return None
         else:
             return str
 
-    def _get_self_arg(self, win_id, param, args):
+    def _get_objreg(self, *, win_id, name, scope):
-        """Get the self argument for a function call.
+        """Get an object from the objreg."""
-
+        if scope == 'global':
-        Arguments:
-            win_id: The window id this command should be executed in.
-            param: The count parameter.
-            args: The positional argument list. Gets modified directly.
-        """
-        assert param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
-        if self._scope == 'global':
             tab_id = None
             win_id = None
-        elif self._scope == 'tab':
+        elif scope == 'tab':
             tab_id = 'current'
-        elif self._scope == 'window':
+        elif scope == 'window':
             tab_id = None
         else:
-            raise ValueError("Invalid scope {}!".format(self._scope))
+            raise ValueError("Invalid scope {}!".format(scope))
-        obj = objreg.get(self._instance, scope=self._scope, window=win_id,
+        return objreg.get(name, scope=scope, window=win_id, tab=tab_id)
-                         tab=tab_id)
-        args.append(obj)
 
-    def _get_count_arg(self, param, args, kwargs):
+    def _add_special_arg(self, *, value, param, args, kwargs):
-        """Add the count argument to a function call.
+        """Add a special argument value to a function call.
 
         Arguments:
-            param: The count parameter.
+            value: The value to add.
+            param: The parameter being filled.
             args: The positional argument list. Gets modified directly.
             kwargs: The keyword argument dict. Gets modified directly.
         """
         if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD:
-            if self._count is not None:
+            args.append(value)
-                args.append(self._count)
-            else:
-                args.append(param.default)
         elif param.kind == inspect.Parameter.KEYWORD_ONLY:
-            if self._count is not None:
+            kwargs[param.name] = value
-                kwargs[param.name] = self._count
         else:
             raise TypeError("{}: invalid parameter type {} for argument "
                             "{!r}!".format(self.name, param.kind, param.name))
 
-    def _get_win_id_arg(self, win_id, param, args, kwargs):
+    def _add_count_tab(self, *, win_id, param, args, kwargs):
-        """Add the win_id argument to a function call.
+        """Add the count_tab widget argument."""
+        tabbed_browser = self._get_objreg(
+            win_id=win_id, name='tabbed-browser', scope='window')
 
-        Arguments:
+        if self._count is None:
-            win_id: The window ID to add.
+            tab = tabbed_browser.widget.currentWidget()
-            param: The count parameter.
+        elif 1 <= self._count <= tabbed_browser.widget.count():
-            args: The positional argument list. Gets modified directly.
+            cmdutils.check_overflow(self._count + 1, 'int')
-            kwargs: The keyword argument dict. Gets modified directly.
+            tab = tabbed_browser.widget.widget(self._count - 1)
-        """
-        if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD:
-            args.append(win_id)
-        elif param.kind == inspect.Parameter.KEYWORD_ONLY:
-            kwargs[param.name] = win_id
         else:
-            raise TypeError("{}: invalid parameter type {} for argument "
+            tab = None
-                            "{!r}!".format(self.name, param.kind, param.name))
+
+        self._add_special_arg(value=tab, param=param, args=args,
+                              kwargs=kwargs)
 
     def _get_param_value(self, param):
         """Get the converted value for an inspect.Parameter."""
@@ -428,6 +431,55 @@ class Command:
 
         return value
 
+    def _handle_special_call_arg(self, *, pos, param, win_id, args, kwargs):
+        """Check whether the argument is special, and if so, fill it in.
+
+        Args:
+            pos: The position of the argument.
+            param: The argparse.Parameter.
+            win_id: The window ID the command is run in.
+            args/kwargs: The args/kwargs to fill.
+
+        Return:
+            True if it was a special arg, False otherwise.
+        """
+        arg_info = self.get_arg_info(param)
+        if pos == 0 and self._instance is not None:
+            assert param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD
+            self_value = self._get_objreg(win_id=win_id, name=self._instance,
+                                          scope=self._scope)
+            self._add_special_arg(value=self_value, param=param,
+                                  args=args, kwargs=kwargs)
+            return True
+        elif arg_info.value == usertypes.CommandValue.count:
+            if self._count is None:
+                assert param.default is not inspect.Parameter.empty
+                value = param.default
+            else:
+                value = self._count
+            self._add_special_arg(value=value, param=param,
+                                  args=args, kwargs=kwargs)
+            return True
+        elif arg_info.value == usertypes.CommandValue.win_id:
+            self._add_special_arg(value=win_id, param=param,
+                                  args=args, kwargs=kwargs)
+            return True
+        elif arg_info.value == usertypes.CommandValue.cur_tab:
+            tab = self._get_objreg(win_id=win_id, name='tab', scope='tab')
+            self._add_special_arg(value=tab, param=param,
+                                  args=args, kwargs=kwargs)
+            return True
+        elif arg_info.value == usertypes.CommandValue.count_tab:
+            self._add_count_tab(win_id=win_id, param=param, args=args,
+                                kwargs=kwargs)
+            return True
+        elif arg_info.value is None:
+            pass
+        else:
+            raise utils.Unreachable(arg_info)
+
+        return False
+
     def _get_call_args(self, win_id):
         """Get arguments for a function call.
 
@@ -442,20 +494,11 @@ class Command:
         signature = inspect.signature(self.handler)
 
         for i, param in enumerate(signature.parameters.values()):
-            arg_info = self.get_arg_info(param)
+            if self._handle_special_call_arg(pos=i, param=param,
-            if i == 0 and self._instance is not None:
+                                             win_id=win_id, args=args,
-                # Special case for 'self'.
+                                             kwargs=kwargs):
-                self._get_self_arg(win_id, param, args)
-                continue
-            elif arg_info.count:
-                # Special case for count parameter.
-                self._get_count_arg(param, args, kwargs)
-                continue
-            # elif arg_info.win_id:
-            elif arg_info.win_id:
-                # Special case for win_id parameter.
-                self._get_win_id_arg(win_id, param, args, kwargs)
                 continue
+
             value = self._get_param_value(param)
             if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD:
                 args.append(value)
@@ -520,4 +563,14 @@ class Command:
 
     def takes_count(self):
         """Return true iff this command can take a count argument."""
-        return any(arg.count for arg in self._qute_args)
+        return any(info.value in self.COUNT_COMMAND_VALUES
+                   for info in self._qute_args.values())
+
+    def register(self):
+        """Register this command in objects.commands."""
+        log.commands.vdebug(
+            "Registering command {} (from {}:{})".format(
+                self.name, self.handler.__module__, self.handler.__qualname__))
+        if self.name in objects.commands:
+            raise ValueError("{} is already registered!".format(self.name))
+        objects.commands[self.name] = self

qutebrowser/commands/runners.py

@@ -25,10 +25,11 @@ import re
 import attr
 from PyQt5.QtCore import pyqtSlot, QUrl, QObject
 
+from qutebrowser.api import cmdutils
 from qutebrowser.config import config
-from qutebrowser.commands import cmdexc, cmdutils
+from qutebrowser.commands import cmdexc
 from qutebrowser.utils import message, objreg, qtutils, usertypes, utils
-from qutebrowser.misc import split
+from qutebrowser.misc import split, objects
 
 
 last_command = {}
@@ -53,11 +54,14 @@ def _current_url(tabbed_browser):
         if e.reason:
             msg += " ({})".format(e.reason)
         msg += "!"
-        raise cmdexc.CommandError(msg)
+        raise cmdutils.CommandError(msg)
 
 
 def replace_variables(win_id, arglist):
     """Utility function to replace variables like {url} in a list of args."""
+    tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                window=win_id)
+
     variables = {
         'url': lambda: _current_url(tabbed_browser).toString(
             QUrl.FullyEncoded | QUrl.RemovePassword),
@@ -67,13 +71,13 @@ def replace_variables(win_id, arglist):
         'clipboard': utils.get_clipboard,
         'primary': lambda: utils.get_clipboard(selection=True),
     }
+
     for key in list(variables):
         modified_key = '{' + key + '}'
         variables[modified_key] = lambda x=modified_key: x
+
     values = {}
     args = []
-    tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                window=win_id)
 
     def repl_cb(matchobj):
         """Return replacement for given match."""
@@ -90,7 +94,7 @@ def replace_variables(win_id, arglist):
             # "{url}" from clipboard is not expanded)
             args.append(repl_pattern.sub(repl_cb, arg))
     except utils.ClipboardError as e:
-        raise cmdexc.CommandError(e)
+        raise cmdutils.CommandError(e)
     return args
 
 
@@ -190,7 +194,7 @@ class CommandParser:
             cmdstr = self._completion_match(cmdstr)
 
         try:
-            cmd = cmdutils.cmd_dict[cmdstr]
+            cmd = objects.commands[cmdstr]
         except KeyError:
             if not fallback:
                 raise cmdexc.NoSuchCommandError(
@@ -217,7 +221,7 @@ class CommandParser:
         Return:
             cmdstr modified to the matching completion or unmodified
         """
-        matches = [cmd for cmd in sorted(cmdutils.cmd_dict, key=len)
+        matches = [cmd for cmd in sorted(objects.commands, key=len)
                    if cmdstr in cmd]
         if len(matches) == 1:
             cmdstr = matches[0]

qutebrowser/completion/completer.py

@@ -23,7 +23,8 @@ import attr
 from PyQt5.QtCore import pyqtSlot, QObject, QTimer
 
 from qutebrowser.config import config
-from qutebrowser.commands import cmdutils, runners
+from qutebrowser.commands import runners
+from qutebrowser.misc import objects
 from qutebrowser.utils import log, utils, debug
 from qutebrowser.completion.models import miscmodels
 
@@ -92,7 +93,7 @@ class Completer(QObject):
             log.completion.debug('Starting command completion')
             return miscmodels.command
         try:
-            cmd = cmdutils.cmd_dict[before_cursor[0]]
+            cmd = objects.commands[before_cursor[0]]
         except KeyError:
             log.completion.debug("No completion for unknown command: {}"
                                  .format(before_cursor[0]))
@@ -170,7 +171,7 @@ class Completer(QObject):
         before, center, after = self._partition()
         log.completion.debug("Changing {} to '{}'".format(center, text))
         try:
-            maxsplit = cmdutils.cmd_dict[before[0]].maxsplit
+            maxsplit = objects.commands[before[0]].maxsplit
         except (KeyError, IndexError):
             maxsplit = None
         if maxsplit is None:

qutebrowser/completion/completiondelegate.py

@@ -166,11 +166,11 @@ class CompletionItemDelegate(QStyledItemDelegate):
         self._painter.save()
 
         if self._opt.state & QStyle.State_Selected:
-            color = config.val.colors.completion.item.selected.fg
+            color = config.cache['colors.completion.item.selected.fg']
         elif not self._opt.state & QStyle.State_Enabled:
-            color = config.val.colors.completion.category.fg
+            color = config.cache['colors.completion.category.fg']
         else:
-            colors = config.val.colors.completion.fg
+            colors = config.cache['colors.completion.fg']
             # if multiple colors are set, use different colors per column
             color = colors[col % len(colors)]
         self._painter.setPen(color)
@@ -212,11 +212,11 @@ class CompletionItemDelegate(QStyledItemDelegate):
             view = self.parent()
             pattern = view.pattern
             columns_to_filter = index.model().columns_to_filter(index)
-            self._doc.setPlainText(self._opt.text)
             if index.column() in columns_to_filter and pattern:
                 pat = re.escape(pattern).replace(r'\ ', r'|')
                 _Highlighter(self._doc, pat,
                              config.val.colors.completion.match.fg)
+            self._doc.setPlainText(self._opt.text)
         else:
             self._doc.setHtml(
                 '<span style="font: {};">{}</span>'.format(

qutebrowser/completion/completionwidget.py

@@ -29,7 +29,7 @@ from PyQt5.QtCore import pyqtSlot, pyqtSignal, Qt, QItemSelectionModel, QSize
 from qutebrowser.config import config
 from qutebrowser.completion import completiondelegate
 from qutebrowser.utils import utils, usertypes, debug, log, objreg
-from qutebrowser.commands import cmdexc, cmdutils
+from qutebrowser.api import cmdutils
 
 
 class CompletionView(QTreeView):
@@ -251,8 +251,8 @@ class CompletionView(QTreeView):
                     status.command_history_prev()
                     return
                 else:
-                    raise cmdexc.CommandError("Can't combine --history with "
+                    raise cmdutils.CommandError("Can't combine --history with "
-                                              "{}!".format(which))
+                                                "{}!".format(which))
 
         if not self._active:
             return
@@ -394,7 +394,7 @@ class CompletionView(QTreeView):
         """Delete the current completion item."""
         index = self.currentIndex()
         if not index.isValid():
-            raise cmdexc.CommandError("No item selected!")
+            raise cmdutils.CommandError("No item selected!")
         self.model().delete_cur_item(index)
 
     @cmdutils.register(instance='completion',
@@ -411,6 +411,6 @@ class CompletionView(QTreeView):
         if not text:
             index = self.currentIndex()
             if not index.isValid():
-                raise cmdexc.CommandError("No item selected!")
+                raise cmdutils.CommandError("No item selected!")
             text = self.model().data(index)
         utils.set_clipboard(text, selection=sel)

qutebrowser/completion/models/completionmodel.py

@@ -22,7 +22,7 @@
 from PyQt5.QtCore import Qt, QModelIndex, QAbstractItemModel
 
 from qutebrowser.utils import log, qtutils
-from qutebrowser.commands import cmdexc
+from qutebrowser.api import cmdutils
 
 
 class CompletionModel(QAbstractItemModel):
@@ -224,7 +224,7 @@ class CompletionModel(QAbstractItemModel):
         cat = self._cat_from_idx(parent)
         assert cat, "CompletionView sent invalid index for deletion"
         if not cat.delete_func:
-            raise cmdexc.CommandError("Cannot delete this item.")
+            raise cmdutils.CommandError("Cannot delete this item.")
 
         data = [cat.data(cat.index(index.row(), i))
                 for i in range(cat.columnCount())]

qutebrowser/completion/models/histcategory.py

@@ -74,9 +74,10 @@ class HistoryCategory(QSqlQueryModel):
 
         # build a where clause to match all of the words in any order
         # given the search term "a b", the WHERE clause would be:
-        # ((url || title) LIKE '%a%') AND ((url || title) LIKE '%b%')
+        # ((url || ' ' || title) LIKE '%a%') AND
+        # ((url || ' ' || title) LIKE '%b%')
         where_clause = ' AND '.join(
-            "(url || title) LIKE :{} escape '\\'".format(i)
+            "(url || ' ' || title) LIKE :{} escape '\\'".format(i)
             for i in range(len(words)))
 
         # replace ' in timestamp-format to avoid breaking the query

qutebrowser/completion/models/util.py

@@ -20,7 +20,7 @@
 """Utility functions for completion models."""
 
 from qutebrowser.utils import objreg, usertypes
-from qutebrowser.commands import cmdutils
+from qutebrowser.misc import objects
 
 
 def get_cmd_completions(info, include_hidden, include_aliases, prefix=''):
@@ -34,10 +34,10 @@ def get_cmd_completions(info, include_hidden, include_aliases, prefix=''):
 
     Return: A list of tuples of form (name, description, bindings).
     """
-    assert cmdutils.cmd_dict
+    assert objects.commands
     cmdlist = []
     cmd_to_keys = info.keyconf.get_reverse_bindings_for('normal')
-    for obj in set(cmdutils.cmd_dict.values()):
+    for obj in set(objects.commands.values()):
         hide_debug = obj.debug and not objreg.get('args').debug
         hide_mode = (usertypes.KeyMode.normal not in obj.modes and
                      not include_hidden)

qutebrowser/components/__init__.py

@@ -0,0 +1,20 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""qutebrowser "extensions" which only use the qutebrowser.api API."""

qutebrowser/components/adblock.py

@@ -24,19 +24,22 @@ import os.path
 import functools
 import posixpath
 import zipfile
+import logging
+import typing
+import pathlib
 
-from qutebrowser.browser import downloads
+from PyQt5.QtCore import QUrl
-from qutebrowser.config import config
+
-from qutebrowser.utils import objreg, standarddir, log, message
+from qutebrowser.api import (cmdutils, hook, config, message, downloads,
-from qutebrowser.commands import cmdutils
+                             interceptor, apitypes)
 
 
-def _guess_zip_filename(zf):
+logger = logging.getLogger('misc')
-    """Guess which file to use inside a zip file.
+_host_blocker = typing.cast('HostBlocker', None)
 
-    Args:
+
-        zf: A ZipFile instance.
+def _guess_zip_filename(zf: zipfile.ZipFile) -> str:
-    """
+    """Guess which file to use inside a zip file."""
     files = zf.namelist()
     if len(files) == 1:
         return files[0]
@@ -47,7 +50,7 @@ def _guess_zip_filename(zf):
     raise FileNotFoundError("No hosts file found in zip")
 
 
-def get_fileobj(byte_io):
+def get_fileobj(byte_io: typing.IO[bytes]) -> typing.IO[bytes]:
     """Get a usable file object to read the hosts file from."""
     byte_io.seek(0)  # rewind downloaded file
     if zipfile.is_zipfile(byte_io):
@@ -60,24 +63,20 @@ def get_fileobj(byte_io):
     return byte_io
 
 
-def _is_whitelisted_url(url):
+def _is_whitelisted_url(url: QUrl) -> bool:
-    """Check if the given URL is on the adblock whitelist.
+    """Check if the given URL is on the adblock whitelist."""
-
-    Args:
-        url: The URL to check as QUrl.
-    """
     for pattern in config.val.content.host_blocking.whitelist:
         if pattern.matches(url):
             return True
     return False
 
 
-class _FakeDownload:
+class _FakeDownload(downloads.TempDownload):
 
     """A download stub to use on_download_finished with local files."""
 
-    def __init__(self, fileobj):
+    def __init__(self,  # pylint: disable=super-init-not-called
-        self.basename = os.path.basename(fileobj.name)
+                 fileobj: typing.IO[bytes]) -> None:
         self.fileobj = fileobj
         self.successful = True
 
@@ -93,37 +92,46 @@ class HostBlocker:
         _done_count: How many files have been read successfully.
         _local_hosts_file: The path to the blocked-hosts file.
         _config_hosts_file: The path to a blocked-hosts in ~/.config
+        _has_basedir: Whether a custom --basedir is set.
     """
 
-    def __init__(self):
+    def __init__(self, *, data_dir: pathlib.Path, config_dir: pathlib.Path,
-        self._blocked_hosts = set()
+                 has_basedir: bool = False) -> None:
-        self._config_blocked_hosts = set()
+        self._has_basedir = has_basedir
-        self._in_progress = []
+        self._blocked_hosts = set()  # type: typing.Set[str]
+        self._config_blocked_hosts = set()  # type: typing.Set[str]
+        self._in_progress = []  # type: typing.List[downloads.TempDownload]
         self._done_count = 0
 
-        data_dir = standarddir.data()
+        self._local_hosts_file = str(data_dir / 'blocked-hosts')
-        self._local_hosts_file = os.path.join(data_dir, 'blocked-hosts')
+        self.update_files()
-        self._update_files()
 
-        config_dir = standarddir.config()
+        self._config_hosts_file = str(config_dir / 'blocked-hosts')
-        self._config_hosts_file = os.path.join(config_dir, 'blocked-hosts')
 
-        config.instance.changed.connect(self._update_files)
+    def _is_blocked(self, request_url: QUrl,
-
+                    first_party_url: QUrl = None) -> bool:
-    def is_blocked(self, url, first_party_url=None):
+        """Check whether the given request is blocked."""
-        """Check if the given URL (as QUrl) is blocked."""
         if first_party_url is not None and not first_party_url.isValid():
             first_party_url = None
-        if not config.instance.get('content.host_blocking.enabled',
+
-                                   url=first_party_url):
+        if not config.get('content.host_blocking.enabled',
+                          url=first_party_url):
             return False
 
-        host = url.host()
+        host = request_url.host()
         return ((host in self._blocked_hosts or
                  host in self._config_blocked_hosts) and
-                not _is_whitelisted_url(url))
+                not _is_whitelisted_url(request_url))
 
-    def _read_hosts_file(self, filename, target):
+    def filter_request(self, info: interceptor.Request) -> None:
+        """Block the given request if necessary."""
+        if self._is_blocked(request_url=info.request_url,
+                            first_party_url=info.first_party_url):
+            logger.info("Request to {} blocked by host blocker."
+                        .format(info.request_url.host()))
+            info.block()
+
+    def _read_hosts_file(self, filename: str, target: typing.Set[str]) -> bool:
         """Read hosts from the given filename.
 
         Args:
@@ -141,11 +149,11 @@ class HostBlocker:
                 for line in f:
                     target.add(line.strip())
         except (OSError, UnicodeDecodeError):
-            log.misc.exception("Failed to read host blocklist!")
+            logger.exception("Failed to read host blocklist!")
 
         return True
 
-    def read_hosts(self):
+    def read_hosts(self) -> None:
         """Read hosts from the existing blocked-hosts file."""
         self._blocked_hosts = set()
 
@@ -156,24 +164,17 @@ class HostBlocker:
                                       self._blocked_hosts)
 
         if not found:
-            args = objreg.get('args')
             if (config.val.content.host_blocking.lists and
-                    args.basedir is None and
+                    not self._has_basedir and
                     config.val.content.host_blocking.enabled):
                 message.info("Run :adblock-update to get adblock lists.")
 
-    @cmdutils.register(instance='host-blocker')
+    def adblock_update(self) -> None:
-    def adblock_update(self):
+        """Update the adblock block lists."""
-        """Update the adblock block lists.
-
-        This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded
-        host lists and re-reads `~/.config/qutebrowser/blocked-hosts`.
-        """
         self._read_hosts_file(self._config_hosts_file,
                               self._config_blocked_hosts)
         self._blocked_hosts = set()
         self._done_count = 0
-        download_manager = objreg.get('qtnetwork-download-manager')
         for url in config.val.content.host_blocking.lists:
             if url.scheme() == 'file':
                 filename = url.toLocalFile()
@@ -184,16 +185,12 @@ class HostBlocker:
                 else:
                     self._import_local(filename)
             else:
-                fobj = io.BytesIO()
+                download = downloads.download_temp(url)
-                fobj.name = 'adblock: ' + url.host()
-                target = downloads.FileObjDownloadTarget(fobj)
-                download = download_manager.get(url, target=target,
-                                                auto_remove=True)
                 self._in_progress.append(download)
                 download.finished.connect(
                     functools.partial(self._on_download_finished, download))
 
-    def _import_local(self, filename):
+    def _import_local(self, filename: str) -> None:
         """Adds the contents of a file to the blocklist.
 
         Args:
@@ -209,24 +206,24 @@ class HostBlocker:
         self._in_progress.append(download)
         self._on_download_finished(download)
 
-    def _parse_line(self, line):
+    def _parse_line(self, raw_line: bytes) -> bool:
         """Parse a line from a host file.
 
         Args:
-            line: The bytes object to parse.
+            raw_line: The bytes object to parse.
 
         Returns:
             True if parsing succeeded, False otherwise.
         """
-        if line.startswith(b'#'):
+        if raw_line.startswith(b'#'):
             # Ignoring comments early so we don't have to care about
             # encoding errors in them.
             return True
 
         try:
-            line = line.decode('utf-8')
+            line = raw_line.decode('utf-8')
         except UnicodeDecodeError:
-            log.misc.error("Failed to decode: {!r}".format(line))
+            logger.error("Failed to decode: {!r}".format(raw_line))
             return False
 
         # Remove comments
@@ -257,14 +254,11 @@ class HostBlocker:
 
         return True
 
-    def _merge_file(self, byte_io):
+    def _merge_file(self, byte_io: io.BytesIO) -> None:
         """Read and merge host files.
 
         Args:
             byte_io: The BytesIO object of the completed download.
-
-        Return:
-            A set of the merged hosts.
         """
         error_count = 0
         line_count = 0
@@ -282,12 +276,12 @@ class HostBlocker:
             if not ok:
                 error_count += 1
 
-        log.misc.debug("{}: read {} lines".format(byte_io.name, line_count))
+        logger.debug("{}: read {} lines".format(byte_io.name, line_count))
         if error_count > 0:
             message.error("adblock: {} read errors for {}".format(
                 error_count, byte_io.name))
 
-    def _on_lists_downloaded(self):
+    def _on_lists_downloaded(self) -> None:
         """Install block lists after files have been downloaded."""
         with open(self._local_hosts_file, 'w', encoding='utf-8') as f:
             for host in sorted(self._blocked_hosts):
@@ -295,8 +289,7 @@ class HostBlocker:
             message.info("adblock: Read {} hosts from {} sources.".format(
                 len(self._blocked_hosts), self._done_count))
 
-    @config.change_filter('content.host_blocking.lists')
+    def update_files(self) -> None:
-    def _update_files(self):
         """Update files when the config changed."""
         if not config.val.content.host_blocking.lists:
             try:
@@ -304,13 +297,13 @@ class HostBlocker:
             except FileNotFoundError:
                 pass
             except OSError as e:
-                log.misc.exception("Failed to delete hosts file: {}".format(e))
+                logger.exception("Failed to delete hosts file: {}".format(e))
 
-    def _on_download_finished(self, download):
+    def _on_download_finished(self, download: downloads.TempDownload) -> None:
         """Check if all downloads are finished and if so, trigger reading.
 
         Arguments:
-            download: The finished DownloadItem.
+            download: The finished download.
         """
         self._in_progress.remove(download)
         if download.successful:
@@ -323,4 +316,32 @@ class HostBlocker:
             try:
                 self._on_lists_downloaded()
             except OSError:
-                log.misc.exception("Failed to write host block list!")
+                logger.exception("Failed to write host block list!")
+
+
+@cmdutils.register()
+def adblock_update() -> None:
+    """Update the adblock block lists.
+
+    This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded
+    host lists and re-reads `~/.config/qutebrowser/blocked-hosts`.
+    """
+    # FIXME: As soon as we can register instances again, we should move this
+    # back to the class.
+    _host_blocker.adblock_update()
+
+
+@hook.config_changed('content.host_blocking.lists')
+def on_config_changed() -> None:
+    _host_blocker.update_files()
+
+
+@hook.init()
+def init(context: apitypes.InitContext) -> None:
+    """Initialize the host blocker."""
+    global _host_blocker
+    _host_blocker = HostBlocker(data_dir=context.data_dir,
+                                config_dir=context.config_dir,
+                                has_basedir=context.args.basedir is not None)
+    _host_blocker.read_hosts()
+    interceptor.register(_host_blocker.filter_request)

qutebrowser/components/caretcommands.py

@@ -0,0 +1,211 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Commands related to caret browsing."""
+
+
+from qutebrowser.api import cmdutils, apitypes
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_next_line(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the next line.
+
+    Args:
+        count: How many lines to move.
+    """
+    tab.caret.move_to_next_line(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_prev_line(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the prev line.
+
+    Args:
+        count: How many lines to move.
+    """
+    tab.caret.move_to_prev_line(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_next_char(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the next char.
+
+    Args:
+        count: How many lines to move.
+    """
+    tab.caret.move_to_next_char(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_prev_char(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the previous char.
+
+    Args:
+        count: How many chars to move.
+    """
+    tab.caret.move_to_prev_char(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_end_of_word(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the end of the word.
+
+    Args:
+        count: How many words to move.
+    """
+    tab.caret.move_to_end_of_word(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_next_word(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the next word.
+
+    Args:
+        count: How many words to move.
+    """
+    tab.caret.move_to_next_word(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_prev_word(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the previous word.
+
+    Args:
+        count: How many words to move.
+    """
+    tab.caret.move_to_prev_word(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+def move_to_start_of_line(tab: apitypes.Tab) -> None:
+    """Move the cursor or selection to the start of the line."""
+    tab.caret.move_to_start_of_line()
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+def move_to_end_of_line(tab: apitypes.Tab) -> None:
+    """Move the cursor or selection to the end of line."""
+    tab.caret.move_to_end_of_line()
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_start_of_next_block(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the start of next block.
+
+    Args:
+        count: How many blocks to move.
+    """
+    tab.caret.move_to_start_of_next_block(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_start_of_prev_block(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the start of previous block.
+
+    Args:
+        count: How many blocks to move.
+    """
+    tab.caret.move_to_start_of_prev_block(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_end_of_next_block(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the end of next block.
+
+    Args:
+        count: How many blocks to move.
+    """
+    tab.caret.move_to_end_of_next_block(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+@cmdutils.argument('count', value=cmdutils.Value.count)
+def move_to_end_of_prev_block(tab: apitypes.Tab, count: int = 1) -> None:
+    """Move the cursor or selection to the end of previous block.
+
+    Args:
+        count: How many blocks to move.
+    """
+    tab.caret.move_to_end_of_prev_block(count)
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+def move_to_start_of_document(tab: apitypes.Tab) -> None:
+    """Move the cursor or selection to the start of the document."""
+    tab.caret.move_to_start_of_document()
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+def move_to_end_of_document(tab: apitypes.Tab) -> None:
+    """Move the cursor or selection to the end of the document."""
+    tab.caret.move_to_end_of_document()
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+def toggle_selection(tab: apitypes.Tab) -> None:
+    """Toggle caret selection mode."""
+    tab.caret.toggle_selection()
+
+
+@cmdutils.register(modes=[cmdutils.KeyMode.caret])
+@cmdutils.argument('tab', value=cmdutils.Value.cur_tab)
+def drop_selection(tab: apitypes.Tab) -> None:
+    """Drop selection and keep selection mode enabled."""
+    tab.caret.drop_selection()
+
+
+@cmdutils.register()
+@cmdutils.argument('tab_obj', value=cmdutils.Value.cur_tab)
+def follow_selected(tab_obj: apitypes.Tab, *, tab: bool = False) -> None:
+    """Follow the selected text.
+
+    Args:
+        tab: Load the selected link in a new tab.
+    """
+    try:
+        tab_obj.caret.follow_selected(tab=tab)
+    except apitypes.WebTabError as e:
+        raise cmdutils.CommandError(str(e))