Merge remote-tracking branch 'upstream/master' into jay/remote-pintab-width

MANIFEST.in

@@ -27,6 +27,7 @@ exclude scripts/asciidoc2html.py
 exclude doc/notes
 recursive-exclude doc *.asciidoc
 include doc/qutebrowser.1.asciidoc
+include doc/changelog.asciidoc
 prune tests
 prune qutebrowser/3rdparty
 prune misc/requirements

doc/help/configuring.asciidoc

@@ -88,7 +88,9 @@ Two global objects are pre-defined when running `config.py`: `c` and `config`.
 Changing settings
 ~~~~~~~~~~~~~~~~~
 
-`c` is a shorthand object to easily set settings like this:
+While you can set settings using the `config.set()` method (which is explained
+in the next section), it's easier to use the `c` shorthand object to easily set
+settings like this:
 
 .config.py:
 [source,python]
@@ -136,6 +138,8 @@ If you want to set settings based on their name as a string, use the
 .config.py:
 [source,python]
 ----
+# Equivalent to:
+# c.content.javascript.enabled = False
 config.set('content.javascript.enabled', False)
 ----
 
@@ -143,6 +147,8 @@ To read a setting, use the `config.get` method:
 
 [source,python]
 ----
+# Equivalent to:
+# color = c.colors.completion.fg
 color = config.get('colors.completion.fg')
 ----
 
@@ -198,17 +204,52 @@ config.bind(',v', 'spawn mpv {url}')
 To suppress loading of any default keybindings, you can set
 `c.bindings.default = {}`.
 
-Prevent loading `autoconfig.yml`
+Loading `autoconfig.yml`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~
 
-If you want all customization done via `:set`, `:bind` and `:unbind` to be
+By default, all customization done via `:set`, `:bind` and `:unbind` is
-temporary, you can suppress loading `autoconfig.yml` in your `config.py` by
+temporary as soon as a `config.py` exists. The settings done that way are always
-doing:
+saved in the `autoconfig.yml` file, but you'll need to explicitly load it in
+your `config.py` by doing:
 
 .config.py:
 [source,python]
 ----
-config.load_autoconfig = False
+config.load_autoconfig()
+----
+
+If you do so at the top of your file, your `config.py` settings will take
+precedence as they overwrite the settings done in `autoconfig.yml`.
+
+Importing other modules
+~~~~~~~~~~~~~~~~~~~~~~~
+
+You can import any module from the
+https://docs.python.org/3/library/index.html[Python standard library] (e.g.
+`import os.path`), as well as any module installed in the environment
+qutebrowser is run with.
+
+If you have an `utils.py` file in your qutebrowser config folder, you can import
+that via `import utils` as well.
+
+While it's in some cases possible to import code from the qutebrowser
+installation, doing so is unsupported and discouraged.
+
+Getting the config directory
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you need to get the qutebrowser config directory, you can do so by reading
+`config.configdir`. Similarily, you can get the qutebrowser data directory via
+`config.datadir`.
+
+This gives you a https://docs.python.org/3/library/pathlib.html[`pathlib.Path`
+object], on which you can use `/` to add more directory parts, or `str(...)` to
+get a string:
+
+.config.py:
+[source,python]
+----
+print(str(config.configdir / 'config.py')
 ----
 
 Handling errors
@@ -221,3 +262,106 @@ qutebrowser tries to display errors which are easy to understand even for people
 who are not used to writing Python. If you see a config error which you find
 confusing or you think qutebrowser could handle better, please
 https://github.com/qutebrowser/qutebrowser/issues[open an issue]!
+
+Recipes
+~~~~~~~
+
+Reading a YAML file
+^^^^^^^^^^^^^^^^^^^
+
+To read a YAML config like this:
+
+.config.yml:
+----
+tabs.position: left
+tabs.show: switching
+----
+
+You can use:
+
+.config.py:
+[source,python]
+----
+import yaml
+
+with (config.configdir / 'config.yml').open() as f:
+    yaml_data = yaml.load(f)
+
+for k, v in yaml_data.items():
+    config.set(k, v)
+----
+
+Reading a nested YAML file
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To read a YAML file with nested values like this:
+
+.colors.yml:
+----
+colors:
+  statusbar:
+    normal:
+      bg: lime
+      fg: black
+    url:
+      fg: red
+----
+
+You can use:
+
+.config.py:
+[source,python]
+----
+import yaml
+
+with (config.configdir / 'colors.yml').open() as f:
+    yaml_data = yaml.load(f)
+
+def dict_attrs(obj, path=''):
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            yield from dict_attrs(v, '{}.{}'.format(path, k) if path else k)
+    else:
+        yield path, obj
+
+for k, v in dict_attrs(yaml_data):
+    config.set(k, v)
+----
+
+Note that this won't work for values which are dictionaries.
+
+Binding chained commands
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you have a lot of chained comamnds you want to bind, you can write a helper
+to do so:
+
+[source,python]
+----
+def bind_chained(key, *commands, force=False):
+    config.bind(key, ' ;; '.join(commands), force=force)
+
+bind_chained('<Escape>', 'clear-keychain', 'search', force=True)
+----
+
+Avoiding flake8 errors
+^^^^^^^^^^^^^^^^^^^^^^
+
+If you use an editor with flake8 integration which complains about `c` and `config` being undefined, you can use:
+
+[source,python]
+----
+c = c  # noqa: F821
+config = config  # noqa: F821
+----
+
+For type annotation support (note that those imports aren't guaranteed to be
+stable across qutebrowser versions):
+
+[source,python]
+----
+from qutebrowser.config.configfiles import ConfigAPI  # noqa: F401
+from qutebrowser.config.config import ConfigContainer  # noqa: F401
+config = config  # type: ConfigAPI # noqa: F821
+c = c  # type: ConfigContainer # noqa: F821
+----

doc/help/index.asciidoc

@@ -7,7 +7,7 @@ Documentation
 The following help pages are currently available:
 
 * link:../quickstart.html[Quick start guide]
-* link:../doc.html[Frequently asked questions]
+* link:../faq.html[Frequently asked questions]
 * link:../changelog.html[Change Log]
 * link:commands.html[Documentation of commands]
 * link:configuring.html[Configuring qutebrowser]

doc/help/settings.asciidoc

@@ -201,7 +201,7 @@
 |<<input.partial_timeout,input.partial_timeout>>|Timeout (in milliseconds) for partially typed key bindings.
 |<<input.rocker_gestures,input.rocker_gestures>>|Enable Opera-like mouse rocker gestures.
 |<<input.spatial_navigation,input.spatial_navigation>>|Enable Spatial Navigation.
-|<<keyhint.blacklist,keyhint.blacklist>>|Keychains that shouldn\'t be shown in the keyhint dialog.
+|<<keyhint.blacklist,keyhint.blacklist>>|Keychains that shouldn't be shown in the keyhint dialog.
 |<<keyhint.delay,keyhint.delay>>|Time from pressing a key to seeing the keyhint dialog (ms).
 |<<messages.timeout,messages.timeout>>|Time (in ms) to show messages in the statusbar for.
 |<<messages.unfocused,messages.unfocused>>|Show messages in unfocused windows.
@@ -283,7 +283,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[backend]]
 === backend
@@ -626,6 +626,7 @@ Default:
 This setting can be used to map keys to other keys.
 When the key used as dictionary-key is pressed, the binding for the key used as dictionary-value is invoked instead.
 This is useful for global remappings of keys, for example to map Ctrl-[ to Escape.
+Note that when a key is bound (via `bindings.default` or `bindings.commands`), the mapping is ignored.
 
 Type: <<types,Dict>>
 
@@ -1341,7 +1342,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[completion.timestamp_format]]
 === completion.timestamp_format
@@ -1401,7 +1402,7 @@ For more information about the feature, please refer to: http://webkit.org/blog/
 
 Type: <<types,Int>>
 
-Default: empty
+Default: +pass:[0]+
 
 This setting is only available with the QtWebKit backend.
 
@@ -1465,7 +1466,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 This setting is only available with the QtWebKit backend.
 
@@ -1496,7 +1497,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 This setting is only available with the QtWebKit backend.
 
@@ -1627,7 +1628,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[content.images]]
 === content.images
@@ -1667,7 +1668,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[content.javascript.can_close_tabs]]
 === content.javascript.can_close_tabs
@@ -1680,7 +1681,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 This setting is only available with the QtWebKit backend.
 
@@ -1695,7 +1696,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[content.javascript.enabled]]
 === content.javascript.enabled
@@ -1736,7 +1737,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[content.javascript.prompt]]
 === content.javascript.prompt
@@ -1775,7 +1776,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[content.local_storage]]
 === content.local_storage
@@ -1841,7 +1842,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 This setting is only available with the QtWebKit backend.
 
@@ -1856,7 +1857,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[content.print_element_backgrounds]]
 === content.print_element_backgrounds
@@ -1884,7 +1885,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[content.proxy]]
 === content.proxy
@@ -1953,7 +1954,7 @@ Default: +pass:[true]+
 [[content.xss_auditing]]
 === content.xss_auditing
 Whether load requests should be monitored for cross-site scripting attempts.
-Suspicious scripts will be blocked and reported in the inspector\'s JavaScript console. Enabling this feature might have an impact on performance.
+Suspicious scripts will be blocked and reported in the inspector's JavaScript console. Enabling this feature might have an impact on performance.
 
 Type: <<types,Bool>>
 
@@ -1962,7 +1963,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[downloads.location.directory]]
 === downloads.location.directory
@@ -2142,7 +2143,7 @@ Default: +pass:[8pt monospace]+
 [[fonts.monospace]]
 === fonts.monospace
 Default monospace fonts.
-Whenever "monospace" is used in a font setting, it\'s replaced with the fonts listed here.
+Whenever "monospace" is used in a font setting, it's replaced with the fonts listed here.
 
 Type: <<types,Font>>
 
@@ -2242,7 +2243,7 @@ The hard minimum font size.
 
 Type: <<types,Int>>
 
-Default: empty
+Default: +pass:[0]+
 
 [[fonts.web.size.minimum_logical]]
 === fonts.web.size.minimum_logical
@@ -2273,7 +2274,7 @@ A timeout (in milliseconds) to ignore normal-mode key bindings after a successfu
 
 Type: <<types,Int>>
 
-Default: empty
+Default: +pass:[0]+
 
 [[hints.border]]
 === hints.border
@@ -2403,7 +2404,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[history_gap_interval]]
 === history_gap_interval
@@ -2466,7 +2467,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[input.insert_mode.plugins]]
 === input.insert_mode.plugins
@@ -2479,7 +2480,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[input.links_included_in_focus_chain]]
 === input.links_included_in_focus_chain
@@ -2515,7 +2516,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[input.spatial_navigation]]
 === input.spatial_navigation
@@ -2529,11 +2530,11 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[keyhint.blacklist]]
 === keyhint.blacklist
-Keychains that shouldn\'t be shown in the keyhint dialog.
+Keychains that shouldn't be shown in the keyhint dialog.
 Globs are supported, so `;*` will blacklist all keychains starting with `;`. Use `*` to disable keyhints.
 
 Type: <<types,List>>
@@ -2568,7 +2569,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[new_instance_open_target]]
 === new_instance_open_target
@@ -2645,7 +2646,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[scrolling.smooth]]
 === scrolling.smooth
@@ -2659,7 +2660,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[session_default_name]]
 === session_default_name
@@ -2681,7 +2682,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[statusbar.padding]]
 === statusbar.padding
@@ -2692,8 +2693,8 @@ Type: <<types,Padding>>
 Default: 
 
 - +pass:[bottom]+: +pass:[1]+
-- +pass:[left]+: empty
+- +pass:[left]+: +pass:[0]+
-- +pass:[right]+: empty
+- +pass:[right]+: +pass:[0]+
 - +pass:[top]+: +pass:[1]+
 
 [[statusbar.position]]
@@ -2720,7 +2721,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[tabs.close_mouse_button]]
 === tabs.close_mouse_button
@@ -2767,7 +2768,7 @@ Type: <<types,Padding>>
 Default: 
 
 - +pass:[bottom]+: +pass:[2]+
-- +pass:[left]+: empty
+- +pass:[left]+: +pass:[0]+
 - +pass:[right]+: +pass:[4]+
 - +pass:[top]+: +pass:[2]+
 
@@ -2838,10 +2839,10 @@ Type: <<types,Padding>>
 
 Default: 
 
-- +pass:[bottom]+: empty
+- +pass:[bottom]+: +pass:[0]+
 - +pass:[left]+: +pass:[5]+
 - +pass:[right]+: +pass:[5]+
-- +pass:[top]+: empty
+- +pass:[top]+: +pass:[0]+
 
 [[tabs.position]]
 === tabs.position
@@ -2906,7 +2907,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[tabs.title.alignment]]
 === tabs.title.alignment
@@ -3069,7 +3070,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 [[window.title_format]]
 === window.title_format
@@ -3143,7 +3144,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: empty
+Default: +pass:[false]+
 
 This setting is only available with the QtWebKit backend.
 
@@ -3157,7 +3158,7 @@ This setting is only available with the QtWebKit backend.
 When setting from a string, `1`, `yes`, `on` and `true` count as true, while `0`, `no`, `off` and `false` count as false (case-insensitive).
 |BoolAsk|Like `Bool`, but `ask` is allowed as additional value.
 |ColorSystem|The color system to use for color interpolation.
-|Command|Base class for a command value with arguments.
+|Command|A qutebrowser command with arguments.
 |ConfirmQuit|Whether to display a confirmation when the window is closed.
 |Dict|A dictionary of values.
 

misc/qutebrowser.spec

@@ -5,6 +5,7 @@ import os
 
 sys.path.insert(0, os.getcwd())
 from scripts import setupcommon
+from qutebrowser import utils
 
 block_cipher = None
 
@@ -30,9 +31,9 @@ def get_data_files():
 setupcommon.write_git_file()
 
 
-if os.name == 'nt':
+if utils.is_windows:
     icon = 'icons/qutebrowser.ico'
-elif sys.platform == 'darwin':
+elif utils.is_mac:
     icon = 'icons/qutebrowser.icns'
 else:
     icon = None

pytest.ini

@@ -25,6 +25,7 @@ markers =
     this: Used to mark tests during development
     no_invalid_lines: Don't fail on unparseable lines in end2end tests
     issue2478: Tests which are broken on Windows with QtWebEngine, https://github.com/qutebrowser/qutebrowser/issues/2478
+    fake_os: Fake utils.is_* to a fake operating system
 qt_log_level_fail = WARNING
 qt_log_ignore =
     ^SpellCheck: .*

qutebrowser/app.py

@@ -43,7 +43,8 @@ import qutebrowser
 import qutebrowser.resources
 from qutebrowser.completion.models import miscmodels
 from qutebrowser.commands import cmdutils, runners, cmdexc
-from qutebrowser.config import config, websettings, configexc, configfiles
+from qutebrowser.config import (config, websettings, configexc, configfiles,
+                                configinit)
 from qutebrowser.browser import (urlmarks, adblock, history, browsertab,
                                  downloads)
 from qutebrowser.browser.network import proxy
@@ -77,7 +78,7 @@ def run(args):
     standarddir.init(args)
 
     log.init.debug("Initializing config...")
-    config.early_init(args)
+    configinit.early_init(args)
 
     global qApp
     qApp = Application(args)
@@ -393,7 +394,7 @@ def _init_modules(args, crash_handler):
     log.init.debug("Initializing save manager...")
     save_manager = savemanager.SaveManager(qApp)
     objreg.register('save-manager', save_manager)
-    config.late_init(save_manager)
+    configinit.late_init(save_manager)
 
     log.init.debug("Initializing network...")
     networkmanager.init()
@@ -762,7 +763,7 @@ class Application(QApplication):
         """
         self._last_focus_object = None
 
-        qt_args = config.qt_args(args)
+        qt_args = configinit.qt_args(args)
         log.init.debug("Qt arguments: {}, based on {}".format(qt_args, args))
         super().__init__(qt_args)
 

qutebrowser/browser/commands.py

@@ -20,7 +20,6 @@
 """Command dispatcher for TabbedBrowser."""
 
 import os
-import sys
 import os.path
 import shlex
 import functools
@@ -430,7 +429,7 @@ class CommandDispatcher:
             tab.printing.to_printer(diag.printer(), print_callback)
 
         diag = QPrintDialog(tab)
-        if sys.platform == 'darwin':
+        if utils.is_mac:
             # For some reason we get a segfault when using open() on macOS
             ret = diag.exec_()
             if ret == QDialog.Accepted:

qutebrowser/browser/downloads.py

@@ -174,7 +174,7 @@ def transform_path(path):
 
     Returns None if the path is invalid on the current platform.
     """
-    if sys.platform != "win32":
+    if not utils.is_windows:
         return path
     path = utils.expand_windows_drive(path)
     # Drive dependent working directories are not supported, e.g.

qutebrowser/browser/history.py

@@ -26,7 +26,7 @@ from PyQt5.QtCore import pyqtSlot, QUrl, QTimer
 
 from qutebrowser.commands import cmdutils, cmdexc
 from qutebrowser.utils import (utils, objreg, log, usertypes, message,
-                               debug, standarddir)
+                               debug, standarddir, qtutils)
 from qutebrowser.misc import objects, sql
 
 
@@ -144,8 +144,10 @@ class WebHistory(sql.SqlTable):
         Args:
             url: URL string to delete.
         """
-        self.delete('url', url)
+        qurl = QUrl(url)
-        self.completion.delete('url', url)
+        qtutils.ensure_valid(qurl)
+        self.delete('url', self._format_url(qurl))
+        self.completion.delete('url', self._format_completion_url(qurl))
 
     @pyqtSlot(QUrl, QUrl, str)
     def add_from_tab(self, url, requested_url, title):
@@ -250,10 +252,7 @@ class WebHistory(sql.SqlTable):
                 except ValueError as ex:
                     message.error('Failed to import history: {}'.format(ex))
                 else:
-                    bakpath = path + '.bak'
+                    self._write_backup(path)
-                    message.info('History import complete. Moving {} to {}'
-                                 .format(path, bakpath))
-                    os.rename(path, bakpath)
 
         # delay to give message time to appear before locking down for import
         message.info('Converting {} to sqlite...'.format(path))
@@ -285,6 +284,16 @@ class WebHistory(sql.SqlTable):
         self.insert_batch(data)
         self.completion.insert_batch(completion_data, replace=True)
 
+    def _write_backup(self, path):
+        bak = path + '.bak'
+        message.info('History import complete. Appending {} to {}'
+                     .format(path, bak))
+        with open(path, 'r', encoding='utf-8') as infile:
+            with open(bak, 'a', encoding='utf-8') as outfile:
+                for line in infile:
+                    outfile.write('\n' + line)
+        os.remove(path)
+
     def _format_url(self, url):
         return url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
 

qutebrowser/browser/webengine/webenginesettings.py

@@ -28,7 +28,6 @@ Module attributes:
 """
 
 import os
-import sys
 import ctypes
 import ctypes.util
 
@@ -207,7 +206,7 @@ def init(args):
 
     # WORKAROUND for
     # https://bugs.launchpad.net/ubuntu/+source/python-qt4/+bug/941826
-    if sys.platform == 'linux':
+    if utils.is_linux:
         ctypes.CDLL(ctypes.util.find_library("GL"), mode=ctypes.RTLD_GLOBAL)
 
     _init_profiles()

qutebrowser/browser/webkit/network/networkmanager.py

@@ -58,7 +58,7 @@ def _is_secure_cipher(cipher):
         # https://codereview.qt-project.org/#/c/75943/
         return False
     # OpenSSL should already protect against this in a better way
-    elif cipher.keyExchangeMethod() == 'DH' and os.name == 'nt':
+    elif cipher.keyExchangeMethod() == 'DH' and utils.is_windows:
         # https://weakdh.org/
         return False
     elif cipher.encryptionMethod().upper().startswith('RC4'):

qutebrowser/browser/webkit/webkittab.py

@@ -19,7 +19,6 @@
 
 """Wrapper over our (QtWebKit) WebView."""
 
-import sys
 import functools
 import xml.etree.ElementTree
 
@@ -223,11 +222,11 @@ class WebKitCaret(browsertab.AbstractCaret):
     def move_to_end_of_word(self, count=1):
         if not self.selection_enabled:
             act = [QWebPage.MoveToNextWord]
-            if sys.platform == 'win32':  # pragma: no cover
+            if utils.is_windows:  # pragma: no cover
                 act.append(QWebPage.MoveToPreviousChar)
         else:
             act = [QWebPage.SelectNextWord]
-            if sys.platform == 'win32':  # pragma: no cover
+            if utils.is_windows:  # pragma: no cover
                 act.append(QWebPage.SelectPreviousChar)
         for _ in range(count):
             for a in act:
@@ -236,11 +235,11 @@ class WebKitCaret(browsertab.AbstractCaret):
     def move_to_next_word(self, count=1):
         if not self.selection_enabled:
             act = [QWebPage.MoveToNextWord]
-            if sys.platform != 'win32':  # pragma: no branch
+            if not utils.is_windows:  # pragma: no branch
                 act.append(QWebPage.MoveToNextChar)
         else:
             act = [QWebPage.SelectNextWord]
-            if sys.platform != 'win32':  # pragma: no branch
+            if not utils.is_windows:  # pragma: no branch
                 act.append(QWebPage.SelectNextChar)
         for _ in range(count):
             for a in act:

qutebrowser/browser/webkit/webview.py

@@ -19,8 +19,6 @@
 
 """The main browser widgets."""
 
-import sys
-
 from PyQt5.QtCore import pyqtSignal, pyqtSlot, Qt, QUrl
 from PyQt5.QtGui import QPalette
 from PyQt5.QtWidgets import QStyleFactory
@@ -57,7 +55,7 @@ class WebView(QWebView):
 
     def __init__(self, *, win_id, tab_id, tab, private, parent=None):
         super().__init__(parent)
-        if sys.platform == 'darwin':
+        if utils.is_mac:
             # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-42948
             # See https://github.com/qutebrowser/qutebrowser/issues/462
             self.setStyle(QStyleFactory.create('Fusion'))

qutebrowser/commands/userscripts.py

@@ -25,7 +25,7 @@ import tempfile
 
 from PyQt5.QtCore import pyqtSignal, pyqtSlot, QObject, QSocketNotifier
 
-from qutebrowser.utils import message, log, objreg, standarddir
+from qutebrowser.utils import message, log, objreg, standarddir, utils
 from qutebrowser.commands import runners
 from qutebrowser.config import config
 from qutebrowser.misc import guiprocess
@@ -406,9 +406,9 @@ def run_async(tab, cmd, *args, win_id, env, verbose=False):
                                 window=win_id)
     commandrunner = runners.CommandRunner(win_id, parent=tabbed_browser)
 
-    if os.name == 'posix':
+    if utils.is_posix:
         runner = _POSIXUserscriptRunner(tabbed_browser)
-    elif os.name == 'nt':  # pragma: no cover
+    elif utils.is_windows:  # pragma: no cover
         runner = _WindowsUserscriptRunner(tabbed_browser)
     else:  # pragma: no cover
         raise UnsupportedError

qutebrowser/completion/completer.py

@@ -154,6 +154,9 @@ class Completer(QObject):
                     "partitioned: {} '{}' {}".format(prefix, center, postfix))
                 return prefix, center, postfix
 
+        # We should always return above
+        assert False, parts
+
     @pyqtSlot(str)
     def on_selection_changed(self, text):
         """Change the completed part if a new item was selected.

qutebrowser/completion/models/configmodel.py

@@ -21,7 +21,7 @@
 
 from qutebrowser.config import configdata, configexc
 from qutebrowser.completion.models import completionmodel, listcategory, util
-from qutebrowser.commands import cmdutils
+from qutebrowser.commands import runners
 
 
 def option(*, info):
@@ -71,8 +71,8 @@ def bind(key, *, info):
     cmd_text = info.keyconf.get_command(key, 'normal')
 
     if cmd_text:
-        cmd_name = cmd_text.split(' ')[0]
+        parser = runners.CommandParser()
-        cmd = cmdutils.cmd_dict.get(cmd_name)
+        cmd = parser.parse(cmd_text).cmd
         data = [(cmd_text, cmd.desc, key)]
         model.add_category(listcategory.ListCategory("Current", data))
 

qutebrowser/config/config.py

@@ -19,19 +19,16 @@
 
 """Configuration storage and config-related utilities."""
 
-import sys
 import copy
 import contextlib
 import functools
 
 from PyQt5.QtCore import pyqtSignal, pyqtSlot, QObject, QUrl
-from PyQt5.QtWidgets import QMessageBox
 
-from qutebrowser.config import configdata, configexc, configtypes, configfiles
+from qutebrowser.config import configdata, configexc, configtypes
-from qutebrowser.utils import (utils, objreg, message, log, usertypes, jinja,
+from qutebrowser.utils import utils, objreg, message, log, jinja
-                               qtutils)
+from qutebrowser.misc import objects
-from qutebrowser.misc import objects, msgbox, earlyinit
+from qutebrowser.commands import cmdexc, cmdutils
-from qutebrowser.commands import cmdexc, cmdutils, runners
 from qutebrowser.completion.models import configmodel
 
 # An easy way to access the config from other code via config.val.foo
@@ -40,9 +37,7 @@ instance = None
 key_instance = None
 
 # Keeping track of all change filters to validate them later.
-_change_filters = []
+change_filters = []
-# Errors which happened during init, so we can show a message box.
-_init_errors = []
 
 
 class change_filter:  # pylint: disable=invalid-name
@@ -68,7 +63,7 @@ class change_filter:  # pylint: disable=invalid-name
         """
         self._option = option
         self._function = function
-        _change_filters.append(self)
+        change_filters.append(self)
 
     def validate(self):
         """Make sure the configured option or prefix exists.
@@ -178,19 +173,6 @@ class KeyConfig:
     def bind(self, key, command, *, mode, force=False, save_yaml=False):
         """Add a new binding from key to command."""
         key = self._prepare(key, mode)
-
-        parser = runners.CommandParser()
-        try:
-            results = parser.parse_all(command)
-        except cmdexc.Error as e:
-            raise configexc.KeybindingError("Invalid command: {}".format(e))
-
-        for result in results:  # pragma: no branch
-            try:
-                result.cmd.validate_mode(usertypes.KeyMode[mode])
-            except cmdexc.PrerequisitesError as e:
-                raise configexc.KeybindingError(str(e))
-
         log.keyboard.vdebug("Adding binding {} -> {} in mode {}.".format(
             key, command, mode))
         if key in self.get_bindings_for(mode) and not force:
@@ -293,14 +275,17 @@ class ConfigCommands:
 
         # Use the next valid value from values, or the first if the current
         # value does not appear in the list
-        old_value = self._config.get_str(option)
+        old_value = self._config.get_obj(option, mutable=False)
+        opt = self._config.get_opt(option)
+        values = [opt.typ.from_str(val) for val in values]
+
         try:
-            idx = values.index(str(old_value))
+            idx = values.index(old_value)
             idx = (idx + 1) % len(values)
             value = values[idx]
         except ValueError:
             value = values[0]
-        self._config.set_str(option, value, save_yaml=not temp)
+        self._config.set_obj(option, value, save_yaml=not temp)
 
     @contextlib.contextmanager
     def _handle_config_error(self):
@@ -408,7 +393,7 @@ class Config(QObject):
     def read_yaml(self):
         """Read the YAML settings from self._yaml."""
         self._yaml.load()
-        for name, value in self._yaml.values.items():
+        for name, value in self._yaml:
             self._set_value(self.get_opt(name), value)
 
     def get_opt(self, name):
@@ -462,7 +447,7 @@ class Config(QObject):
         """
         self._set_value(self.get_opt(name), value)
         if save_yaml:
-            self._yaml.values[name] = value
+            self._yaml[name] = value
 
     def set_str(self, name, value, *, save_yaml=False):
         """Set the given setting from a string.
@@ -476,7 +461,7 @@ class Config(QObject):
                                  value))
         self._set_value(opt, converted)
         if save_yaml:
-            self._yaml.values[name] = converted
+            self._yaml[name] = converted
 
     def update_mutables(self, *, save_yaml=False):
         """Update mutable settings if they changed.
@@ -647,114 +632,3 @@ class StyleSheetObserver(QObject):
         self._obj.setStyleSheet(qss)
         if update:
             instance.changed.connect(self._update_stylesheet)
-
-
-def early_init(args):
-    """Initialize the part of the config which works without a QApplication."""
-    configdata.init()
-
-    yaml_config = configfiles.YamlConfig()
-
-    global val, instance, key_instance
-    instance = Config(yaml_config=yaml_config)
-    val = ConfigContainer(instance)
-    key_instance = KeyConfig(instance)
-
-    for cf in _change_filters:
-        cf.validate()
-
-    configtypes.Font.monospace_fonts = val.fonts.monospace
-
-    config_commands = ConfigCommands(instance, key_instance)
-    objreg.register('config-commands', config_commands)
-
-    config_api = None
-
-    try:
-        config_api = configfiles.read_config_py()
-        # Raised here so we get the config_api back.
-        if config_api.errors:
-            raise configexc.ConfigFileErrors('config.py', config_api.errors)
-    except configexc.ConfigFileErrors as e:
-        log.config.exception("Error while loading config.py")
-        _init_errors.append(e)
-
-    try:
-        if getattr(config_api, 'load_autoconfig', True):
-            try:
-                instance.read_yaml()
-            except configexc.ConfigFileErrors as e:
-                raise  # caught in outer block
-            except configexc.Error as e:
-                desc = configexc.ConfigErrorDesc("Error", e)
-                raise configexc.ConfigFileErrors('autoconfig.yml', [desc])
-    except configexc.ConfigFileErrors as e:
-        log.config.exception("Error while loading config.py")
-        _init_errors.append(e)
-
-    configfiles.init()
-
-    objects.backend = get_backend(args)
-    earlyinit.init_with_backend(objects.backend)
-
-
-def get_backend(args):
-    """Find out what backend to use based on available libraries."""
-    try:
-        import PyQt5.QtWebKit  # pylint: disable=unused-variable
-    except ImportError:
-        webkit_available = False
-    else:
-        webkit_available = qtutils.is_new_qtwebkit()
-
-    str_to_backend = {
-        'webkit': usertypes.Backend.QtWebKit,
-        'webengine': usertypes.Backend.QtWebEngine,
-    }
-
-    if args.backend is not None:
-        return str_to_backend[args.backend]
-    elif val.backend != 'auto':
-        return str_to_backend[val.backend]
-    elif webkit_available:
-        return usertypes.Backend.QtWebKit
-    else:
-        return usertypes.Backend.QtWebEngine
-
-
-def late_init(save_manager):
-    """Initialize the rest of the config after the QApplication is created."""
-    global _init_errors
-    for err in _init_errors:
-        errbox = msgbox.msgbox(parent=None,
-                               title="Error while reading config",
-                               text=err.to_html(),
-                               icon=QMessageBox.Warning,
-                               plain_text=False)
-        errbox.exec_()
-    _init_errors = []
-
-    instance.init_save_manager(save_manager)
-    configfiles.state.init_save_manager(save_manager)
-
-
-def qt_args(namespace):
-    """Get the Qt QApplication arguments based on an argparse namespace.
-
-    Args:
-        namespace: The argparse namespace.
-
-    Return:
-        The argv list to be passed to Qt.
-    """
-    argv = [sys.argv[0]]
-
-    if namespace.qt_flag is not None:
-        argv += ['--' + flag[0] for flag in namespace.qt_flag]
-
-    if namespace.qt_arg is not None:
-        for name, value in namespace.qt_arg:
-            argv += ['--' + name, value]
-
-    argv += ['--' + arg for arg in val.qt_args]
-    return argv

qutebrowser/config/configdata.yml

@@ -562,10 +562,12 @@ content.xss_auditing:
   desc: >-
     Whether load requests should be monitored for cross-site scripting attempts.
 
-    Suspicious scripts will be blocked and reported in the inspector\'s
+    Suspicious scripts will be blocked and reported in the inspector's
     JavaScript console. Enabling this feature might have an impact on
     performance.
 
+# emacs: '
+
 ## completion
 
 completion.cmd_history_max_items:
@@ -917,11 +919,13 @@ keyhint.blacklist:
       name: String
   default: []
   desc: >-
-    Keychains that shouldn\'t be shown in the keyhint dialog.
+    Keychains that shouldn't be shown in the keyhint dialog.
 
     Globs are supported, so `;*` will blacklist all keychains starting with `;`.
     Use `*` to disable keyhints.
 
+# emacs: '
+
 keyhint.delay:
   type:
     name: Int
@@ -1727,9 +1731,11 @@ fonts.monospace:
   desc: >-
     Default monospace fonts.
 
-    Whenever "monospace" is used in a font setting, it\'s replaced with the
+    Whenever "monospace" is used in a font setting, it's replaced with the
     fonts listed here.
 
+# emacs: '
+
 fonts.completion.entry:
   default: 8pt monospace
   type: Font
@@ -1896,6 +1902,9 @@ bindings.key_mappings:
     This is useful for global remappings of keys, for example to map Ctrl-[ to
     Escape.
 
+    Note that when a key is bound (via `bindings.default` or
+    `bindings.commands`), the mapping is ignored.
+
 bindings.default:
   default:
     normal:

qutebrowser/config/configexc.py

@@ -94,6 +94,12 @@ class ConfigErrorDesc:
     def __str__(self):
         return '{}: {}'.format(self.text, self.exception)
 
+    def with_text(self, text):
+        """Get a new ConfigErrorDesc with the given text appended."""
+        return self.__class__(text='{} ({})'.format(self.text, text),
+                              exception=self.exception,
+                              traceback=self.traceback)
+
 
 class ConfigFileErrors(Error):
 

qutebrowser/config/configfiles.py

@@ -19,18 +19,20 @@
 
 """Configuration files residing on disk."""
 
+import pathlib
 import types
 import os.path
+import sys
 import textwrap
 import traceback
 import configparser
 import contextlib
 
 import yaml
-from PyQt5.QtCore import QSettings
+from PyQt5.QtCore import pyqtSignal, QObject, QSettings
 
 import qutebrowser
-from qutebrowser.config import configexc, config
+from qutebrowser.config import configexc, config, configdata
 from qutebrowser.utils import standarddir, utils, qtutils
 
 
@@ -70,7 +72,7 @@ class StateConfig(configparser.ConfigParser):
             self.write(f)
 
 
-class YamlConfig:
+class YamlConfig(QObject):
 
     """A config stored on disk as YAML file.
 
@@ -79,11 +81,14 @@ class YamlConfig:
     """
 
     VERSION = 1
+    changed = pyqtSignal()
 
-    def __init__(self):
+    def __init__(self, parent=None):
+        super().__init__(parent)
         self._filename = os.path.join(standarddir.config(auto=True),
                                       'autoconfig.yml')
-        self.values = {}
+        self._values = {}
+        self._dirty = None
 
     def init_save_manager(self, save_manager):
         """Make sure the config gets saved properly.
@@ -91,11 +96,28 @@ class YamlConfig:
         We do this outside of __init__ because the config gets created before
         the save_manager exists.
         """
-        save_manager.add_saveable('yaml-config', self._save)
+        save_manager.add_saveable('yaml-config', self._save, self.changed)
+
+    def __getitem__(self, name):
+        return self._values[name]
+
+    def __setitem__(self, name, value):
+        self.changed.emit()
+        self._dirty = True
+        self._values[name] = value
+
+    def __contains__(self, name):
+        return name in self._values
+
+    def __iter__(self):
+        return iter(self._values.items())
 
     def _save(self):
-        """Save the changed settings to the YAML file."""
+        """Save the settings to the YAML file if they've changed."""
-        data = {'config_version': self.VERSION, 'global': self.values}
+        if not self._dirty:
+            return
+
+        data = {'config_version': self.VERSION, 'global': self._values}
         with qtutils.savefile_open(self._filename) as f:
             f.write(textwrap.dedent("""
                 # DO NOT edit this file by hand, qutebrowser will overwrite it.
@@ -105,12 +127,12 @@ class YamlConfig:
             utils.yaml_dump(data, f)
 
     def load(self):
-        """Load self.values from the configured YAML file."""
+        """Load configuration from the configured YAML file."""
         try:
             with open(self._filename, 'r', encoding='utf-8') as f:
                 yaml_data = utils.yaml_load(f)
         except FileNotFoundError:
-            return
+            return {}
         except OSError as e:
             desc = configexc.ConfigErrorDesc("While reading", e)
             raise configexc.ConfigFileErrors('autoconfig.yml', [desc])
@@ -136,7 +158,14 @@ class YamlConfig:
                 "'global' object is not a dict")
             raise configexc.ConfigFileErrors('autoconfig.yml', [desc])
 
-        self.values = global_obj
+        # Delete unknown values
+        # (e.g. options which were removed from configdata.yml)
+        for name in list(global_obj):
+            if name not in configdata.DATA:
+                del global_obj[name]
+
+        self._values = global_obj
+        self._dirty = False
 
 
 class ConfigAPI:
@@ -150,20 +179,26 @@ class ConfigAPI:
     Attributes:
         _config: The main Config object to use.
         _keyconfig: The KeyConfig object.
-        load_autoconfig: Whether autoconfig.yml should be loaded.
         errors: Errors which occurred while setting options.
+        configdir: The qutebrowser config directory, as pathlib.Path.
+        datadir: The qutebrowser data directory, as pathlib.Path.
     """
 
     def __init__(self, conf, keyconfig):
         self._config = conf
         self._keyconfig = keyconfig
-        self.load_autoconfig = True
         self.errors = []
+        self.configdir = pathlib.Path(standarddir.config())
+        self.datadir = pathlib.Path(standarddir.data())
 
     @contextlib.contextmanager
     def _handle_error(self, action, name):
         try:
             yield
+        except configexc.ConfigFileErrors as e:
+            for err in e.errors:
+                new_err = err.with_text(e.basename)
+                self.errors.append(new_err)
         except configexc.Error as e:
             text = "While {} '{}'".format(action, name)
             self.errors.append(configexc.ConfigErrorDesc(text, e))
@@ -172,6 +207,10 @@ class ConfigAPI:
         """Do work which needs to be done after reading config.py."""
         self._config.update_mutables()
 
+    def load_autoconfig(self):
+        with self._handle_error('reading', 'autoconfig.yml'):
+            read_autoconfig()
+
     def get(self, name):
         with self._handle_error('getting', name):
             return self._config.get_obj(name)
@@ -182,22 +221,26 @@ class ConfigAPI:
 
     def bind(self, key, command, mode='normal', *, force=False):
         with self._handle_error('binding', key):
+            try:
                 self._keyconfig.bind(key, command, mode=mode, force=force)
+            except configexc.DuplicateKeyError as e:
+                raise configexc.KeybindingError('{} - use force=True to '
+                                                'override!'.format(e))
 
     def unbind(self, key, mode='normal'):
         with self._handle_error('unbinding', key):
             self._keyconfig.unbind(key, mode=mode)
 
 
-def read_config_py(filename=None):
+def read_config_py(filename, raising=False):
-    """Read a config.py file."""
+    """Read a config.py file.
+
+    Arguments;
+        filename: The name of the file to read.
+        raising: Raise exceptions happening in config.py.
+                 This is needed during tests to use pytest's inspection.
+    """
     api = ConfigAPI(config.instance, config.key_instance)
-
-    if filename is None:
-        filename = os.path.join(standarddir.config(), 'config.py')
-        if not os.path.exists(filename):
-            return api
-
     container = config.ConfigContainer(config.instance, configapi=api)
     basename = os.path.basename(filename)
 
@@ -216,7 +259,7 @@ def read_config_py(filename=None):
 
     try:
         code = compile(source, filename, 'exec')
-    except (ValueError, TypeError) as e:
+    except ValueError as e:
         # source contains NUL bytes
         desc = configexc.ConfigErrorDesc("Error while compiling", e)
         raise configexc.ConfigFileErrors(basename, [desc])
@@ -226,14 +269,51 @@ def read_config_py(filename=None):
         raise configexc.ConfigFileErrors(basename, [desc])
 
     try:
+        # Save and restore sys variables
+        with saved_sys_properties():
+            # Add config directory to python path, so config.py can import
+            # other files in logical places
+            config_dir = os.path.dirname(filename)
+            if config_dir not in sys.path:
+                sys.path.insert(0, config_dir)
+
             exec(code, module.__dict__)
     except Exception as e:
+        if raising:
+            raise
         api.errors.append(configexc.ConfigErrorDesc(
             "Unhandled exception",
             exception=e, traceback=traceback.format_exc()))
 
     api.finalize()
-    return api
+
+    if api.errors:
+        raise configexc.ConfigFileErrors('config.py', api.errors)
+
+
+def read_autoconfig():
+    """Read the autoconfig.yml file."""
+    try:
+        config.instance.read_yaml()
+    except configexc.ConfigFileErrors as e:
+        raise  # caught in outer block
+    except configexc.Error as e:
+        desc = configexc.ConfigErrorDesc("Error", e)
+        raise configexc.ConfigFileErrors('autoconfig.yml', [desc])
+
+
+@contextlib.contextmanager
+def saved_sys_properties():
+    """Save various sys properties such as sys.path and sys.modules."""
+    old_path = sys.path.copy()
+    old_modules = sys.modules.copy()
+
+    try:
+        yield
+    finally:
+        sys.path = old_path
+        for module in set(sys.modules).difference(old_modules):
+            del sys.modules[module]
 
 
 def init():

qutebrowser/config/configinit.py

@@ -0,0 +1,134 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Initialization of the configuration."""
+
+import os.path
+import sys
+
+from PyQt5.QtWidgets import QMessageBox
+
+from qutebrowser.config import (config, configdata, configfiles, configtypes,
+                                configexc)
+from qutebrowser.utils import objreg, qtutils, usertypes, log, standarddir
+from qutebrowser.misc import earlyinit, msgbox, objects
+
+
+# Error which happened during init, so we can show a message box.
+_init_errors = None
+
+
+def early_init(args):
+    """Initialize the part of the config which works without a QApplication."""
+    configdata.init()
+
+    yaml_config = configfiles.YamlConfig()
+
+    config.instance = config.Config(yaml_config=yaml_config)
+    config.val = config.ConfigContainer(config.instance)
+    config.key_instance = config.KeyConfig(config.instance)
+    yaml_config.setParent(config.instance)
+
+    for cf in config.change_filters:
+        cf.validate()
+
+    configtypes.Font.monospace_fonts = config.val.fonts.monospace
+
+    config_commands = config.ConfigCommands(config.instance,
+                                            config.key_instance)
+    objreg.register('config-commands', config_commands)
+
+    config_file = os.path.join(standarddir.config(), 'config.py')
+
+    try:
+        if os.path.exists(config_file):
+            configfiles.read_config_py(config_file)
+        else:
+            configfiles.read_autoconfig()
+    except configexc.ConfigFileErrors as e:
+        log.config.exception("Error while loading {}".format(e.basename))
+        global _init_errors
+        _init_errors = e
+
+    configfiles.init()
+
+    objects.backend = get_backend(args)
+    earlyinit.init_with_backend(objects.backend)
+
+
+def get_backend(args):
+    """Find out what backend to use based on available libraries."""
+    try:
+        import PyQt5.QtWebKit  # pylint: disable=unused-variable
+    except ImportError:
+        webkit_available = False
+    else:
+        webkit_available = qtutils.is_new_qtwebkit()
+
+    str_to_backend = {
+        'webkit': usertypes.Backend.QtWebKit,
+        'webengine': usertypes.Backend.QtWebEngine,
+    }
+
+    if args.backend is not None:
+        return str_to_backend[args.backend]
+    elif config.val.backend != 'auto':
+        return str_to_backend[config.val.backend]
+    elif webkit_available:
+        return usertypes.Backend.QtWebKit
+    else:
+        return usertypes.Backend.QtWebEngine
+
+
+def late_init(save_manager):
+    """Initialize the rest of the config after the QApplication is created."""
+    global _init_errors
+    if _init_errors is not None:
+        errbox = msgbox.msgbox(parent=None,
+                               title="Error while reading config",
+                               text=_init_errors.to_html(),
+                               icon=QMessageBox.Warning,
+                               plain_text=False)
+        errbox.exec_()
+    _init_errors = None
+
+    config.instance.init_save_manager(save_manager)
+    configfiles.state.init_save_manager(save_manager)
+
+
+def qt_args(namespace):
+    """Get the Qt QApplication arguments based on an argparse namespace.
+
+    Args:
+        namespace: The argparse namespace.
+
+    Return:
+        The argv list to be passed to Qt.
+    """
+    argv = [sys.argv[0]]
+
+    if namespace.qt_flag is not None:
+        argv += ['--' + flag[0] for flag in namespace.qt_flag]
+
+    if namespace.qt_arg is not None:
+        for name, value in namespace.qt_arg:
+            argv += ['--' + name, value]
+
+    argv += ['--' + arg for arg in config.val.qt_args]
+    return argv

qutebrowser/config/configtypes.py

@@ -59,7 +59,7 @@ from PyQt5.QtCore import QUrl, Qt
 from PyQt5.QtGui import QColor, QFont
 from PyQt5.QtWidgets import QTabWidget, QTabBar
 
-from qutebrowser.commands import cmdutils, runners, cmdexc
+from qutebrowser.commands import cmdutils
 from qutebrowser.config import configexc
 from qutebrowser.utils import standarddir, utils, qtutils, urlutils
 
@@ -257,9 +257,10 @@ class BaseType:
         This currently uses asciidoc syntax.
         """
         utils.unused(indent)  # only needed for Dict/List
-        if not value:
+        str_value = self.to_str(value)
+        if not str_value:
             return 'empty'
-        return '+pass:[{}]+'.format(html.escape(self.to_str(value)))
+        return '+pass:[{}]+'.format(html.escape(str_value))
 
     def complete(self):
         """Return a list of possible values for completion.
@@ -773,33 +774,13 @@ class PercOrInt(_Numeric):
 
 class Command(BaseType):
 
-    """Base class for a command value with arguments."""
+    """A qutebrowser command with arguments.
 
-    # See to_py for details
+    //
-    unvalidated = False
 
-    def to_py(self, value):
+    Since validation is quite tricky here, we don't do so, and instead let
-        self._basic_py_validation(value, str)
+    invalid commands (in bindings/aliases) fail when used.
-        if not value:
+    """
-            return None
-
-        # This requires some trickery, as runners.CommandParser uses
-        # conf.val.aliases, which in turn map to a command again,
-        # leading to an endless recursion.
-        # To fix that, we turn off validating other commands (alias values)
-        # while validating a command.
-        if not Command.unvalidated:
-            Command.unvalidated = True
-            try:
-                parser = runners.CommandParser()
-                try:
-                    parser.parse_all(value)
-                except cmdexc.Error as e:
-                    raise configexc.ValidationError(value, str(e))
-            finally:
-                Command.unvalidated = False
-
-        return value
 
     def complete(self):
         out = []
@@ -807,6 +788,10 @@ class Command(BaseType):
             out.append((cmdname, obj.desc))
         return out
 
+    def to_py(self, value):
+        self._basic_py_validation(value, str)
+        return value
+
 
 class ColorSystem(MappingType):
 

qutebrowser/html/settings.html

@@ -17,6 +17,9 @@ pre { margin: 2px; }
 th, td { border: 1px solid grey; padding: 0px 5px; }
 th { background: lightgrey; }
 th pre { color: grey; text-align: left; }
+input { width: 98%; }
+.setting { width: 75%; }
+.value { width: 25%; text-align: center; }
 .noscript, .noscript-text { color:red; }
 .noscript-text { margin-bottom: 5cm; }
 .option_description { margin: .5ex 0; color: grey; font-size: 80%; font-style: italic; white-space: pre-line; }
@@ -26,15 +29,19 @@ th pre { color: grey; text-align: left; }
 <noscript><h1 class="noscript">View Only</h1><p class="noscript-text">Changing settings requires javascript to be enabled!</p></noscript>
 <header><h1>{{ title }}</h1></header>
 <table>
+    <tr>
+        <th>Setting</th>
+        <th>Value</th>
+    </tr>
   {% for option in configdata.DATA.values() %}
     <tr>
       <!-- FIXME: convert to string properly -->
-      <td>{{ option.name }} (Current: {{ confget(option.name) | string |truncate(100) }})
+      <td class="setting">{{ option.name }} (Current: {{ confget(option.name) | string |truncate(100) }})
         {% if option.description %}
           <p class="option_description">{{ option.description|e }}</p>
         {% endif %}
       </td>
-      <td>
+      <td class="value">
         <input type="text"
           id="input-{{ option.name }}"
           onblur="cset('{{ option.name }}', this.value)"

qutebrowser/keyinput/basekeyparser.py

@@ -122,6 +122,7 @@ class BaseKeyParser(QObject):
             self._debug_log("Ignoring only-modifier keyeevent.")
             return False
 
+        if binding not in self.special_bindings:
             key_mappings = config.val.bindings.key_mappings
             try:
                 binding = key_mappings['<{}>'.format(binding)][1:-1]
@@ -133,26 +134,28 @@ class BaseKeyParser(QObject):
         except KeyError:
             self._debug_log("No special binding found for {}.".format(binding))
             return False
-        count, _command = self._split_count()
+        count, _command = self._split_count(self._keystring)
         self.execute(cmdstr, self.Type.special, count)
         self.clear_keystring()
         return True
 
-    def _split_count(self):
+    def _split_count(self, keystring):
         """Get count and command from the current keystring.
 
+        Args:
+            keystring: The key string to split.
+
         Return:
             A (count, command) tuple.
         """
         if self._supports_count:
-            (countstr, cmd_input) = re.match(r'^(\d*)(.*)',
+            (countstr, cmd_input) = re.match(r'^(\d*)(.*)', keystring).groups()
-                                             self._keystring).groups()
             count = int(countstr) if countstr else None
             if count == 0 and not cmd_input:
-                cmd_input = self._keystring
+                cmd_input = keystring
                 count = None
         else:
-            cmd_input = self._keystring
+            cmd_input = keystring
             count = None
         return count, cmd_input
 
@@ -183,18 +186,17 @@ class BaseKeyParser(QObject):
             self._debug_log("Ignoring, no text char")
             return self.Match.none
 
-        key_mappings = config.val.bindings.key_mappings
+        count, cmd_input = self._split_count(self._keystring + txt)
-        txt = key_mappings.get(txt, txt)
+        match, binding = self._match_key(cmd_input)
-        self._keystring += txt
+        if match == self.Match.none:
-
+            mappings = config.val.bindings.key_mappings
-        count, cmd_input = self._split_count()
+            mapped = mappings.get(txt, None)
-
+            if mapped is not None:
-        if not cmd_input:
+                txt = mapped
-            # Only a count, no command yet, but we handled it
+                count, cmd_input = self._split_count(self._keystring + txt)
-            return self.Match.other
-
                 match, binding = self._match_key(cmd_input)
 
+        self._keystring += txt
         if match == self.Match.definitive:
             self._debug_log("Definitive match for '{}'.".format(
                 self._keystring))
@@ -207,6 +209,8 @@ class BaseKeyParser(QObject):
             self._debug_log("Giving up with '{}', no matches".format(
                 self._keystring))
             self.clear_keystring()
+        elif match == self.Match.other:
+            pass
         else:
             raise AssertionError("Invalid match value {!r}".format(match))
         return match
@@ -223,6 +227,9 @@ class BaseKeyParser(QObject):
                 binding: - None with Match.partial/Match.none.
                          - The found binding with Match.definitive.
         """
+        if not cmd_input:
+            # Only a count, no command yet, but we handled it
+            return (self.Match.other, None)
         # A (cmd_input, binding) tuple (k, v of bindings) or None.
         definitive_match = None
         partial_match = False

qutebrowser/misc/crashsignal.py

@@ -40,7 +40,7 @@ from PyQt5.QtWidgets import QApplication, QDialog
 
 from qutebrowser.commands import cmdutils
 from qutebrowser.misc import earlyinit, crashdialog
-from qutebrowser.utils import usertypes, standarddir, log, objreg, debug
+from qutebrowser.utils import usertypes, standarddir, log, objreg, debug, utils
 
 
 @attr.s
@@ -312,7 +312,7 @@ class SignalHandler(QObject):
         self._orig_handlers[signal.SIGTERM] = signal.signal(
             signal.SIGTERM, self.interrupt)
 
-        if os.name == 'posix' and hasattr(signal, 'set_wakeup_fd'):
+        if utils.is_posix and hasattr(signal, 'set_wakeup_fd'):
             # pylint: disable=import-error,no-member,useless-suppression
             import fcntl
             read_fd, write_fd = os.pipe()

qutebrowser/misc/earlyinit.py

@@ -47,33 +47,25 @@ except ImportError:
 START_TIME = datetime.datetime.now()
 
 
-def _missing_str(name, *, windows=None, pip=None, webengine=False):
+def _missing_str(name, *, webengine=False):
     """Get an error string for missing packages.
 
     Args:
         name: The name of the package.
-        windows: String to be displayed for Windows.
-        pip: pypi package name.
         webengine: Whether this is checking the QtWebEngine package
     """
     blocks = ["Fatal error: <b>{}</b> is required to run qutebrowser but "
               "could not be imported! Maybe it's not installed?".format(name),
               "<b>The error encountered was:</b><br />%ERROR%"]
     lines = ['Please search for the python3 version of {} in your '
-             'distributions packages, or install it via pip.'.format(name)]
+             'distributions packages, or see '
+             'https://github.com/qutebrowser/qutebrowser/blob/master/doc/install.asciidoc'
+             .format(name)]
     blocks.append('<br />'.join(lines))
     if not webengine:
         lines = ['<b>If you installed a qutebrowser package for your '
                  'distribution, please report this as a bug.</b>']
         blocks.append('<br />'.join(lines))
-    if windows is not None:
-        lines = ["<b>On Windows:</b>"]
-        lines += windows.splitlines()
-        blocks.append('<br />'.join(lines))
-    if pip is not None:
-        lines = ["<b>Using pip:</b>"]
-        lines.append("pip3 install {}".format(pip))
-        blocks.append('<br />'.join(lines))
     return '<br /><br />'.join(blocks)
 
 
@@ -142,11 +134,7 @@ def check_pyqt_core():
     try:
         import PyQt5.QtCore  # pylint: disable=unused-variable
     except ImportError as e:
-        text = _missing_str('PyQt5',
+        text = _missing_str('PyQt5')
-                            windows="Use the installer by Riverbank computing "
-                                    "or the standalone qutebrowser exe.<br />"
-                                    "http://www.riverbankcomputing.co.uk/"
-                                    "software/pyqt/download5")
         text = text.replace('<b>', '')
         text = text.replace('</b>', '')
         text = text.replace('<br />', '\n')
@@ -230,7 +218,14 @@ def _check_modules(modules):
                         'Flags not at the start of the expression']
             with log.ignore_py_warnings(
                     category=DeprecationWarning,
-                    message=r'({})'.format('|'.join(messages))):
+                    message=r'({})'.format('|'.join(messages))
+            ), log.ignore_py_warnings(
+                    category=PendingDeprecationWarning,
+                    module='imp'
+            ), log.ignore_py_warnings(
+                    category=ImportWarning,
+                    message=r'Not importing directory .*: missing __init__'
+            ):
                 importlib.import_module(name)
         except ImportError as e:
             _die(text, e)
@@ -239,31 +234,12 @@ def _check_modules(modules):
 def check_libraries():
     """Check if all needed Python libraries are installed."""
     modules = {
-        'pkg_resources':
+        'pkg_resources': _missing_str("pkg_resources/setuptools"),
-            _missing_str("pkg_resources/setuptools",
+        'pypeg2': _missing_str("pypeg2"),
-                         windows="Run   python -m ensurepip."),
+        'jinja2': _missing_str("jinja2"),
-        'pypeg2':
+        'pygments': _missing_str("pygments"),
-            _missing_str("pypeg2",
+        'yaml': _missing_str("PyYAML"),
-                         pip="pypeg2"),
+        'attr': _missing_str("attrs"),
-        'jinja2':
-            _missing_str("jinja2",
-                         windows="Install from http://www.lfd.uci.edu/"
-                                 "~gohlke/pythonlibs/#jinja2 or via pip.",
-                         pip="jinja2"),
-        'pygments':
-            _missing_str("pygments",
-                         windows="Install from http://www.lfd.uci.edu/"
-                                 "~gohlke/pythonlibs/#pygments or via pip.",
-                         pip="pygments"),
-        'yaml':
-            _missing_str("PyYAML",
-                         windows="Use the installers at "
-                                 "http://pyyaml.org/download/pyyaml/ (py3.4) "
-                                 "or Install via pip.",
-                         pip="PyYAML"),
-        'attr':
-            _missing_str("attrs",
-                         pip="attrs"),
         'PyQt5.QtQml': _missing_str("PyQt5.QtQml"),
         'PyQt5.QtSql': _missing_str("PyQt5.QtSql"),
         'PyQt5.QtOpenGL': _missing_str("PyQt5.QtOpenGL"),

qutebrowser/misc/ipc.py

@@ -30,7 +30,7 @@ from PyQt5.QtCore import pyqtSignal, pyqtSlot, QObject, Qt
 from PyQt5.QtNetwork import QLocalSocket, QLocalServer, QAbstractSocket
 
 import qutebrowser
-from qutebrowser.utils import log, usertypes, error, objreg, standarddir
+from qutebrowser.utils import log, usertypes, error, objreg, standarddir, utils
 
 
 CONNECT_TIMEOUT = 100  # timeout for connecting/disconnecting
@@ -51,7 +51,7 @@ def _get_socketname_windows(basedir):
 
 def _get_socketname(basedir):
     """Get a socketname to use."""
-    if os.name == 'nt':  # pragma: no cover
+    if utils.is_windows:  # pragma: no cover
         return _get_socketname_windows(basedir)
 
     parts_to_hash = [getpass.getuser()]
@@ -139,8 +139,6 @@ class IPCServer(QObject):
         _server: A QLocalServer to accept new connections.
         _socket: The QLocalSocket we're currently connected to.
         _socketname: The socketname to use.
-        _socketopts_ok: Set if using setSocketOptions is working with this
-                        OS/Qt version.
         _atime_timer: Timer to update the atime of the socket regularly.
 
     Signals:
@@ -169,7 +167,7 @@ class IPCServer(QObject):
         self._timer.setInterval(READ_TIMEOUT)
         self._timer.timeout.connect(self.on_timeout)
 
-        if os.name == 'nt':  # pragma: no cover
+        if utils.is_windows:  # pragma: no cover
             self._atime_timer = None
         else:
             self._atime_timer = usertypes.Timer(self, 'ipc-atime')
@@ -182,8 +180,7 @@ class IPCServer(QObject):
 
         self._socket = None
         self._old_socket = None
-        self._socketopts_ok = os.name == 'nt'
+        if utils.is_windows:  # pragma: no cover
-        if self._socketopts_ok:  # pragma: no cover
             # If we use setSocketOptions on Unix with Qt < 5.4, we get a
             # NameError while listening...
             log.ipc.debug("Calling setSocketOptions")
@@ -210,7 +207,7 @@ class IPCServer(QObject):
                 raise AddressInUseError(self._server)
             else:
                 raise ListenError(self._server)
-        if not self._socketopts_ok:  # pragma: no cover
+        if not utils.is_windows:  # pragma: no cover
             # If we use setSocketOptions on Unix with Qt < 5.4, we get a
             # NameError while listening.
             # (see b135569d5c6e68c735ea83f42e4baf51f7972281)

qutebrowser/utils/log.py

@@ -409,6 +409,8 @@ def qt_message_handler(msg_type, context, msg):
         # https://codereview.qt-project.org/176831
         "QObject::disconnect: Unexpected null parameter",
     ]
+    # not using utils.is_mac here, because we can't be sure we can successfully
+    # import the utils module here.
     if sys.platform == 'darwin':
         suppressed_msgs += [
             'libpng warning: iCCP: known incorrect sRGB profile',

qutebrowser/utils/standarddir.py

@@ -20,7 +20,6 @@
 """Utilities to get and initialize data/config paths."""
 
 import os
-import sys
 import shutil
 import os.path
 import contextlib
@@ -28,7 +27,7 @@ import contextlib
 from PyQt5.QtCore import QStandardPaths
 from PyQt5.QtWidgets import QApplication
 
-from qutebrowser.utils import log, debug, usertypes, message
+from qutebrowser.utils import log, debug, usertypes, message, utils
 
 # The cached locations
 _locations = {}
@@ -69,7 +68,7 @@ def _init_config(args):
     typ = QStandardPaths.ConfigLocation
     overridden, path = _from_args(typ, args)
     if not overridden:
-        if os.name == 'nt':
+        if utils.is_windows:
             app_data_path = _writable_location(
                 QStandardPaths.AppDataLocation)
             path = os.path.join(app_data_path, 'config')
@@ -80,7 +79,7 @@ def _init_config(args):
     _locations[Location.auto_config] = path
 
     # Override the normal (non-auto) config on macOS
-    if sys.platform == 'darwin':
+    if utils.is_mac:
         overridden, path = _from_args(typ, args)
         if not overridden:  # pragma: no branch
             path = os.path.expanduser('~/.' + APPNAME)
@@ -104,7 +103,7 @@ def _init_data(args):
     typ = QStandardPaths.DataLocation
     overridden, path = _from_args(typ, args)
     if not overridden:
-        if os.name == 'nt':
+        if utils.is_windows:
             app_data_path = _writable_location(QStandardPaths.AppDataLocation)
             path = os.path.join(app_data_path, 'data')
         else:
@@ -114,7 +113,7 @@ def _init_data(args):
 
     # system_data
     _locations.pop(Location.system_data, None)  # Remove old state
-    if sys.platform.startswith('linux'):
+    if utils.is_linux:
         path = '/usr/share/' + APPNAME
         if os.path.exists(path):
             _locations[Location.system_data] = path
@@ -139,7 +138,7 @@ def _init_cache(args):
     typ = QStandardPaths.CacheLocation
     overridden, path = _from_args(typ, args)
     if not overridden:
-        if os.name == 'nt':
+        if utils.is_windows:
             # Local, not Roaming!
             data_path = _writable_location(QStandardPaths.DataLocation)
             path = os.path.join(data_path, 'cache')
@@ -172,7 +171,7 @@ def download():
 
 def _init_runtime(args):
     """Initialize location for runtime data."""
-    if sys.platform.startswith('linux'):
+    if utils.is_linux:
         typ = QStandardPaths.RuntimeLocation
     else:
         # RuntimeLocation is a weird path on macOS and Windows.
@@ -312,9 +311,9 @@ def init(args):
     _init_dirs(args)
     _init_cachedir_tag()
     if args is not None and getattr(args, 'basedir', None) is None:
-        if sys.platform == 'darwin':  # pragma: no cover
+        if utils.is_mac:  # pragma: no cover
             _move_macos()
-        elif os.name == 'nt':  # pragma: no cover
+        elif utils.is_windows:  # pragma: no cover
             _move_windows()
 
 

qutebrowser/utils/utils.py

@@ -19,6 +19,7 @@
 
 """Other utilities which don't fit anywhere else."""
 
+import os
 import io
 import re
 import sys
@@ -49,6 +50,11 @@ from qutebrowser.utils import qtutils, log, debug
 fake_clipboard = None
 log_clipboard = False
 
+is_mac = sys.platform.startswith('darwin')
+is_linux = sys.platform.startswith('linux')
+is_windows = sys.platform.startswith('win')
+is_posix = os.name == 'posix'
+
 
 class ClipboardError(Exception):
 
@@ -377,7 +383,7 @@ def keyevent_to_string(e):
         A name of the key (combination) as a string or
         None if only modifiers are pressed..
     """
-    if sys.platform == 'darwin':
+    if is_mac:
         # Qt swaps Ctrl/Meta on macOS, so we switch it back here so the user
         # can use it in the config as expected. See:
         # https://github.com/qutebrowser/qutebrowser/issues/110

qutebrowser/utils/version.py

@@ -248,12 +248,12 @@ def _os_info():
     """
     lines = []
     releaseinfo = None
-    if sys.platform == 'linux':
+    if utils.is_linux:
         osver = ''
         releaseinfo = _release_info()
-    elif sys.platform == 'win32':
+    elif utils.is_windows:
         osver = ', '.join(platform.win32_ver())
-    elif sys.platform == 'darwin':
+    elif utils.is_mac:
         release, versioninfo, machine = platform.mac_ver()
         if all(not e for e in versioninfo):
             versioninfo = ''

scripts/dev/build_release.py

@@ -36,7 +36,8 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
                                 os.pardir))
 
 import qutebrowser
-from scripts import utils
+from qutebrowser.utils import utils
+from scripts import utils as scriptutils
 # from scripts.dev import update_3rdparty
 
 
@@ -70,7 +71,7 @@ def call_tox(toxenv, *args, python=sys.executable):
 
 def run_asciidoc2html(args):
     """Common buildsteps used for all OS'."""
-    utils.print_title("Running asciidoc2html.py")
+    scriptutils.print_title("Running asciidoc2html.py")
     if args.asciidoc is not None:
         a2h_args = ['--asciidoc'] + args.asciidoc
     else:
@@ -127,7 +128,7 @@ def patch_mac_app():
 
 def build_mac():
     """Build macOS .dmg/.app."""
-    utils.print_title("Cleaning up...")
+    scriptutils.print_title("Cleaning up...")
     for f in ['wc.dmg', 'template.dmg']:
         try:
             os.remove(f)
@@ -135,20 +136,20 @@ def build_mac():
             pass
     for d in ['dist', 'build']:
         shutil.rmtree(d, ignore_errors=True)
-    utils.print_title("Updating 3rdparty content")
+    scriptutils.print_title("Updating 3rdparty content")
     # Currently disabled because QtWebEngine has no pdfjs support
     # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False)
-    utils.print_title("Building .app via pyinstaller")
+    scriptutils.print_title("Building .app via pyinstaller")
     call_tox('pyinstaller', '-r')
-    utils.print_title("Patching .app")
+    scriptutils.print_title("Patching .app")
     patch_mac_app()
-    utils.print_title("Building .dmg")
+    scriptutils.print_title("Building .dmg")
     subprocess.check_call(['make', '-f', 'scripts/dev/Makefile-dmg'])
 
     dmg_name = 'qutebrowser-{}.dmg'.format(qutebrowser.__version__)
     os.rename('qutebrowser.dmg', dmg_name)
 
-    utils.print_title("Running smoke test")
+    scriptutils.print_title("Running smoke test")
 
     try:
         with tempfile.TemporaryDirectory() as tmpdir:
@@ -177,11 +178,11 @@ def patch_windows(out_dir):
 
 def build_windows():
     """Build windows executables/setups."""
-    utils.print_title("Updating 3rdparty content")
+    scriptutils.print_title("Updating 3rdparty content")
     # Currently disabled because QtWebEngine has no pdfjs support
     # update_3rdparty.run(ace=False, pdfjs=True, fancy_dmg=False)
 
-    utils.print_title("Building Windows binaries")
+    scriptutils.print_title("Building Windows binaries")
     parts = str(sys.version_info.major), str(sys.version_info.minor)
     ver = ''.join(parts)
     python_x86 = r'C:\Python{}-32\python.exe'.format(ver)
@@ -194,19 +195,19 @@ def build_windows():
 
     artifacts = []
 
-    utils.print_title("Running pyinstaller 32bit")
+    scriptutils.print_title("Running pyinstaller 32bit")
     _maybe_remove(out_32)
     call_tox('pyinstaller', '-r', python=python_x86)
     shutil.move(out_pyinstaller, out_32)
     patch_windows(out_32)
 
-    utils.print_title("Running pyinstaller 64bit")
+    scriptutils.print_title("Running pyinstaller 64bit")
     _maybe_remove(out_64)
     call_tox('pyinstaller', '-r', python=python_x64)
     shutil.move(out_pyinstaller, out_64)
     patch_windows(out_64)
 
-    utils.print_title("Building installers")
+    scriptutils.print_title("Building installers")
     subprocess.check_call(['makensis.exe',
                            '/DVERSION={}'.format(qutebrowser.__version__),
                            'misc/qutebrowser.nsi'])
@@ -227,12 +228,12 @@ def build_windows():
          'Windows 64bit installer'),
     ]
 
-    utils.print_title("Running 32bit smoke test")
+    scriptutils.print_title("Running 32bit smoke test")
     smoke_test(os.path.join(out_32, 'qutebrowser.exe'))
-    utils.print_title("Running 64bit smoke test")
+    scriptutils.print_title("Running 64bit smoke test")
     smoke_test(os.path.join(out_64, 'qutebrowser.exe'))
 
-    utils.print_title("Zipping 32bit standalone...")
+    scriptutils.print_title("Zipping 32bit standalone...")
     name = 'qutebrowser-{}-windows-standalone-win32'.format(
         qutebrowser.__version__)
     shutil.make_archive(name, 'zip', 'dist', os.path.basename(out_32))
@@ -240,7 +241,7 @@ def build_windows():
                       'application/zip',
                       'Windows 32bit standalone'))
 
-    utils.print_title("Zipping 64bit standalone...")
+    scriptutils.print_title("Zipping 64bit standalone...")
     name = 'qutebrowser-{}-windows-standalone-amd64'.format(
         qutebrowser.__version__)
     shutil.make_archive(name, 'zip', 'dist', os.path.basename(out_64))
@@ -253,7 +254,7 @@ def build_windows():
 
 def build_sdist():
     """Build an sdist and list the contents."""
-    utils.print_title("Building sdist")
+    scriptutils.print_title("Building sdist")
 
     _maybe_remove('dist')
 
@@ -276,10 +277,10 @@ def build_sdist():
 
     assert '.pyc' not in by_ext
 
-    utils.print_title("sdist contents")
+    scriptutils.print_title("sdist contents")
 
     for ext, files in sorted(by_ext.items()):
-        utils.print_subtitle(ext)
+        scriptutils.print_subtitle(ext)
         print('\n'.join(files))
 
     filename = 'qutebrowser-{}.tar.gz'.format(qutebrowser.__version__)
@@ -308,7 +309,7 @@ def github_upload(artifacts, tag):
         tag: The name of the release tag
     """
     import github3
-    utils.print_title("Uploading to github...")
+    scriptutils.print_title("Uploading to github...")
 
     token = read_github_token()
     gh = github3.login(token=token)
@@ -343,7 +344,7 @@ def main():
     parser.add_argument('--upload', help="Tag to upload the release for",
                         nargs=1, required=False, metavar='TAG')
     args = parser.parse_args()
-    utils.change_cwd()
+    scriptutils.change_cwd()
 
     upload_to_pypi = False
 
@@ -353,7 +354,8 @@ def main():
         import github3  # pylint: disable=unused-variable
         read_github_token()
 
-    if os.name == 'nt':
+    run_asciidoc2html(args)
+    if utils.is_windows:
         if sys.maxsize > 2**32:
             # WORKAROUND
             print("Due to a python/Windows bug, this script needs to be run ")
@@ -362,21 +364,24 @@ def main():
             print("See http://bugs.python.org/issue24493 and ")
             print("https://github.com/pypa/virtualenv/issues/774")
             sys.exit(1)
-        run_asciidoc2html(args)
         artifacts = build_windows()
-    elif sys.platform == 'darwin':
+    elif utils.is_mac:
-        run_asciidoc2html(args)
         artifacts = build_mac()
     else:
         artifacts = build_sdist()
         upload_to_pypi = True
 
     if args.upload is not None:
-        utils.print_title("Press enter to release...")
+        scriptutils.print_title("Press enter to release...")
         input()
         github_upload(artifacts, args.upload[0])
         if upload_to_pypi:
             pypi_upload(artifacts)
+    else:
+        print()
+        scriptutils.print_title("Artifacts")
+        for artifact in artifacts:
+            print(artifact)
 
 
 if __name__ == '__main__':

scripts/dev/check_coverage.py

@@ -32,7 +32,8 @@ import attr
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
                                 os.pardir))
 
-from scripts import utils
+from scripts import utils as scriptutils
+from qutebrowser.utils import utils
 
 
 @attr.s
@@ -140,6 +141,8 @@ PERFECT_FILES = [
         'config/configfiles.py'),
     ('tests/unit/config/test_configtypes.py',
         'config/configtypes.py'),
+    ('tests/unit/config/test_configinit.py',
+        'config/configinit.py'),
 
     ('tests/unit/utils/test_qtutils.py',
         'utils/qtutils.py'),
@@ -207,7 +210,7 @@ def _get_filename(filename):
 
 def check(fileobj, perfect_files):
     """Main entry point which parses/checks coverage.xml if applicable."""
-    if sys.platform != 'linux':
+    if not utils.is_linux:
         raise Skipped("on non-Linux system.")
     elif '-k' in sys.argv[1:]:
         raise Skipped("because -k is given.")
@@ -272,7 +275,7 @@ def main_check():
     if messages:
         print()
         print()
-        utils.print_title("Coverage check failed")
+        scriptutils.print_title("Coverage check failed")
         for msg in messages:
             print(msg.text)
         print()
@@ -323,7 +326,7 @@ def main_check_all():
 
 
 def main():
-    utils.change_cwd()
+    scriptutils.change_cwd()
     if '--check-all' in sys.argv:
         return main_check_all()
     else:

scripts/dev/run_vulture.py

@@ -41,7 +41,7 @@ from qutebrowser.browser import qutescheme
 from qutebrowser.config import configtypes
 
 
-def whitelist_generator():
+def whitelist_generator():  # noqa
     """Generator which yields lines to add to a vulture whitelist."""
     # qutebrowser commands
     for cmd in cmdutils.cmd_dict.values():
@@ -108,6 +108,8 @@ def whitelist_generator():
     yield 'qutebrowser.config.configexc.ConfigErrorDesc.traceback'
     yield 'qutebrowser.config.configfiles.ConfigAPI.load_autoconfig'
     yield 'types.ModuleType.c'  # configfiles:read_config_py
+    for name in ['configdir', 'datadir']:
+        yield 'qutebrowser.config.configfiles.ConfigAPI.' + name
 
     yield 'include_aliases'
 

scripts/setupcommon.py

@@ -18,11 +18,9 @@
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
 
-"""Data used by setup.py and scripts/freeze.py."""
+"""Data used by setup.py and the PyInstaller qutebrowser.spec."""
 
 import sys
-import re
-import ast
 import os
 import os.path
 import subprocess
@@ -30,42 +28,16 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir))
 
 
 if sys.hexversion >= 0x03000000:
-    _open = open
+    open_file = open
 else:
     import codecs
-    _open = codecs.open
+    open_file = codecs.open
 
 
 BASEDIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                        os.path.pardir)
 
 
-def read_file(name):
-    """Get the string contained in the file named name."""
-    with _open(name, 'r', encoding='utf-8') as f:
-        return f.read()
-
-
-def _get_constant(name):
-    """Read a __magic__ constant from qutebrowser/__init__.py.
-
-    We don't import qutebrowser here because it can go wrong for multiple
-    reasons. Instead we use re/ast to get the value directly from the source
-    file.
-
-    Args:
-        name: The name of the argument to get.
-
-    Return:
-        The value of the argument.
-    """
-    field_re = re.compile(r'__{}__\s+=\s+(.*)'.format(re.escape(name)))
-    path = os.path.join(BASEDIR, 'qutebrowser', '__init__.py')
-    line = field_re.search(read_file(path)).group(1)
-    value = ast.literal_eval(line)
-    return value
-
-
 def _git_str():
     """Try to find out git version.
 
@@ -95,37 +67,5 @@ def write_git_file():
     if gitstr is None:
         gitstr = ''
     path = os.path.join(BASEDIR, 'qutebrowser', 'git-commit-id')
-    with _open(path, 'w', encoding='ascii') as f:
+    with open_file(path, 'w', encoding='ascii') as f:
         f.write(gitstr)
-
-
-setupdata = {
-    'name': 'qutebrowser',
-    'version': '.'.join(str(e) for e in _get_constant('version_info')),
-    'description': _get_constant('description'),
-    'long_description': read_file('README.asciidoc'),
-    'url': 'https://www.qutebrowser.org/',
-    'requires': ['pypeg2', 'jinja2', 'pygments', 'PyYAML', 'attrs'],
-    'author': _get_constant('author'),
-    'author_email': _get_constant('email'),
-    'license': _get_constant('license'),
-    'classifiers': [
-        'Development Status :: 3 - Alpha',
-        'Environment :: X11 Applications :: Qt',
-        'Intended Audience :: End Users/Desktop',
-        'License :: OSI Approved :: GNU General Public License v3 or later '
-            '(GPLv3+)',
-        'Natural Language :: English',
-        'Operating System :: Microsoft :: Windows',
-        'Operating System :: Microsoft :: Windows :: Windows XP',
-        'Operating System :: Microsoft :: Windows :: Windows 7',
-        'Operating System :: POSIX :: Linux',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        'Topic :: Internet',
-        'Topic :: Internet :: WWW/HTTP',
-        'Topic :: Internet :: WWW/HTTP :: Browsers',
-    ],
-    'keywords': 'pyqt browser web qt webkit',
-}

setup.py

@@ -21,6 +21,8 @@
 
 """setuptools installer script for qutebrowser."""
 
+import re
+import ast
 import os
 import os.path
 
@@ -35,6 +37,32 @@ except NameError:
     BASEDIR = None
 
 
+def read_file(name):
+    """Get the string contained in the file named name."""
+    with common.open_file(name, 'r', encoding='utf-8') as f:
+        return f.read()
+
+
+def _get_constant(name):
+    """Read a __magic__ constant from qutebrowser/__init__.py.
+
+    We don't import qutebrowser here because it can go wrong for multiple
+    reasons. Instead we use re/ast to get the value directly from the source
+    file.
+
+    Args:
+        name: The name of the argument to get.
+
+    Return:
+        The value of the argument.
+    """
+    field_re = re.compile(r'__{}__\s+=\s+(.*)'.format(re.escape(name)))
+    path = os.path.join(BASEDIR, 'qutebrowser', '__init__.py')
+    line = field_re.search(read_file(path)).group(1)
+    value = ast.literal_eval(line)
+    return value
+
+
 try:
     common.write_git_file()
     setuptools.setup(
@@ -42,10 +70,35 @@ try:
         include_package_data=True,
         entry_points={'gui_scripts':
                       ['qutebrowser = qutebrowser.qutebrowser:main']},
-        test_suite='qutebrowser.test',
         zip_safe=True,
         install_requires=['pypeg2', 'jinja2', 'pygments', 'PyYAML', 'attrs'],
-        **common.setupdata
+        name='qutebrowser',
+        version='.'.join(str(e) for e in _get_constant('version_info')),
+        description=_get_constant('description'),
+        long_description=read_file('README.asciidoc'),
+        url='https://www.qutebrowser.org/',
+        author=_get_constant('author'),
+        author_email=_get_constant('email'),
+        license=_get_constant('license'),
+        classifiers=[
+            'Development Status :: 4 - Beta',
+            'Environment :: X11 Applications :: Qt',
+            'Intended Audience :: End Users/Desktop',
+            'License :: OSI Approved :: GNU General Public License v3 or later '
+                '(GPLv3+)',
+            'Natural Language :: English',
+            'Operating System :: Microsoft :: Windows',
+            'Operating System :: POSIX :: Linux',
+            'Operating System :: MacOS',
+            'Operating System :: POSIX :: BSD',
+            'Programming Language :: Python :: 3',
+            'Programming Language :: Python :: 3.5',
+            'Programming Language :: Python :: 3.6',
+            'Topic :: Internet',
+            'Topic :: Internet :: WWW/HTTP',
+            'Topic :: Internet :: WWW/HTTP :: Browsers',
+        ],
+        keywords='pyqt browser web qt webkit qtwebkit qtwebengine',
     )
 finally:
     if BASEDIR is not None:

tests/conftest.py

@@ -35,7 +35,7 @@ from helpers import logfail
 from helpers.logfail import fail_on_logging
 from helpers.messagemock import message_mock
 from helpers.fixtures import *
-from qutebrowser.utils import qtutils, standarddir, usertypes
+from qutebrowser.utils import qtutils, standarddir, usertypes, utils
 from qutebrowser.misc import objects
 
 import qutebrowser.app  # To register commands
@@ -50,18 +50,18 @@ hypothesis.settings.load_profile('default')
 def _apply_platform_markers(config, item):
     """Apply a skip marker to a given item."""
     markers = [
-        ('posix', os.name != 'posix', "Requires a POSIX os"),
+        ('posix', not utils.is_posix, "Requires a POSIX os"),
-        ('windows', os.name != 'nt', "Requires Windows"),
+        ('windows', not utils.is_windows, "Requires Windows"),
-        ('linux', not sys.platform.startswith('linux'), "Requires Linux"),
+        ('linux', not utils.is_linux, "Requires Linux"),
-        ('mac', sys.platform != 'darwin', "Requires macOS"),
+        ('mac', not utils.is_mac, "Requires macOS"),
-        ('not_mac', sys.platform == 'darwin', "Skipped on macOS"),
+        ('not_mac', utils.is_mac, "Skipped on macOS"),
         ('not_frozen', getattr(sys, 'frozen', False),
             "Can't be run when frozen"),
         ('frozen', not getattr(sys, 'frozen', False),
             "Can only run when frozen"),
         ('ci', 'CI' not in os.environ, "Only runs on CI."),
         ('no_ci', 'CI' in os.environ, "Skipped on CI."),
-        ('issue2478', os.name == 'nt' and config.webengine,
+        ('issue2478', utils.is_windows and config.webengine,
          "Broken with QtWebEngine on Windows"),
     ]
 
@@ -181,7 +181,7 @@ def check_display(request):
             request.config.xvfb is not None):
         raise Exception("Xvfb is running on buildbot!")
 
-    if sys.platform == 'linux' and not os.environ.get('DISPLAY', ''):
+    if utils.is_linux and not os.environ.get('DISPLAY', ''):
         raise Exception("No display and no Xvfb available!")
 
 
@@ -193,6 +193,37 @@ def set_backend(monkeypatch, request):
     monkeypatch.setattr(objects, 'backend', backend)
 
 
+@pytest.fixture(autouse=True)
+def apply_fake_os(monkeypatch, request):
+    fake_os = request.node.get_marker('fake_os')
+    if not fake_os:
+        return
+
+    name = fake_os.args[0]
+    mac = False
+    windows = False
+    linux = False
+    posix = False
+
+    if name == 'unknown':
+        pass
+    elif name == 'mac':
+        mac = True
+        posix = True
+    elif name == 'windows':
+        windows = True
+    elif name == 'linux':
+        linux = True
+        posix = True
+    else:
+        raise ValueError("Invalid fake_os {}".format(name))
+
+    monkeypatch.setattr('qutebrowser.utils.utils.is_mac', mac)
+    monkeypatch.setattr('qutebrowser.utils.utils.is_linux', linux)
+    monkeypatch.setattr('qutebrowser.utils.utils.is_windows', windows)
+    monkeypatch.setattr('qutebrowser.utils.utils.is_posix', posix)
+
+
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
 def pytest_runtest_makereport(item, call):
     """Make test information available in fixtures.

tests/end2end/conftest.py

@@ -38,7 +38,7 @@ from end2end.fixtures.webserver import server, server_after_test, ssl_server
 from end2end.fixtures.quteprocess import (quteproc_process, quteproc,
                                           quteproc_new)
 from end2end.fixtures.testprocess import pytest_runtest_makereport
-from qutebrowser.utils import qtutils
+from qutebrowser.utils import qtutils, utils
 
 
 def pytest_configure(config):
@@ -144,7 +144,7 @@ def pytest_collection_modifyitems(config, items):
         ('qtwebengine_flaky', 'Flaky with QtWebEngine', pytest.mark.skipif,
          config.webengine),
         ('qtwebengine_mac_xfail', 'Fails on macOS with QtWebEngine',
-         pytest.mark.xfail, config.webengine and sys.platform == 'darwin'),
+         pytest.mark.xfail, config.webengine and utils.is_mac),
     ]
 
     for item in items:

tests/end2end/features/conftest.py

@@ -31,9 +31,9 @@ import textwrap
 import pytest
 import pytest_bdd as bdd
 
-from qutebrowser.utils import log
+from qutebrowser.utils import log, utils
 from qutebrowser.browser import pdfjs
-from helpers import utils
+from helpers import utils as testutils
 
 
 def _get_echo_exe_path():
@@ -42,8 +42,9 @@ def _get_echo_exe_path():
     Return:
         Path to the "echo"-utility.
     """
-    if sys.platform == "win32":
+    if utils.is_windows:
-        return os.path.join(utils.abs_datapath(), 'userscripts', 'echo.bat')
+        return os.path.join(testutils.abs_datapath(), 'userscripts',
+                            'echo.bat')
     else:
         return 'echo'
 
@@ -255,7 +256,7 @@ def run_command(quteproc, server, tmpdir, command):
         invalid = False
 
     command = command.replace('(port)', str(server.port))
-    command = command.replace('(testdata)', utils.abs_datapath())
+    command = command.replace('(testdata)', testutils.abs_datapath())
     command = command.replace('(tmpdir)', str(tmpdir))
     command = command.replace('(dirsep)', os.sep)
     command = command.replace('(echo-exe)', _get_echo_exe_path())
@@ -349,7 +350,7 @@ def hint(quteproc, args):
 
 @bdd.when(bdd.parsers.parse('I hint with args "{args}" and follow {letter}'))
 def hint_and_follow(quteproc, args, letter):
-    args = args.replace('(testdata)', utils.abs_datapath())
+    args = args.replace('(testdata)', testutils.abs_datapath())
     quteproc.send_cmd(':hint {}'.format(args))
     quteproc.wait_for(message='hints: *')
     quteproc.send_cmd(':follow-hint {}'.format(letter))
@@ -502,7 +503,7 @@ def check_header(quteproc, header, value):
         assert header not in data['headers']
     else:
         actual = data['headers'][header]
-        assert utils.pattern_match(pattern=value, value=actual)
+        assert testutils.pattern_match(pattern=value, value=actual)
 
 
 @bdd.then(bdd.parsers.parse('the page should contain the html "{text}"'))

tests/end2end/features/hints.feature

@@ -362,6 +362,7 @@ Feature: Using hints
         And I set hints.mode to letter
         And I hint with args "--mode number all"
         And I press the key "s"
+        And I wait for "Filtering hints on 's'" in the log
         And I run :follow-hint 1
         Then data/numbers/7.txt should be loaded
 

tests/end2end/features/javascript.feature

@@ -61,6 +61,7 @@ Feature: Javascript stuff
         And I run :jseval if (window.open('about:blank')) { console.log('window opened'); } else { console.log('error while opening window'); }
         Then the javascript message "window opened" should be logged
 
+    @flaky
     Scenario: Opening window without user interaction with javascript.can_open_tabs_automatically set to false
         When I open data/hello.txt
         And I set content.javascript.can_open_tabs_automatically to false

tests/end2end/fixtures/test_testprocess.py

@@ -138,9 +138,9 @@ def test_quitting_process(qtbot, quit_pyproc):
 
 
 def test_quitting_process_expected(qtbot, quit_pyproc):
+    quit_pyproc.exit_expected = True
     with qtbot.waitSignal(quit_pyproc.proc.finished):
         quit_pyproc.start()
-    quit_pyproc.exit_expected = True
     quit_pyproc.after_test()
 
 

tests/end2end/fixtures/testprocess.py

@@ -125,6 +125,7 @@ class Process(QObject):
     Attributes:
         _invalid: A list of lines which could not be parsed.
         _data: A list of parsed lines.
+        _started: Whether the process was ever started.
         proc: The QProcess for the underlying process.
         exit_expected: Whether the process is expected to quit.
 
@@ -140,11 +141,12 @@ class Process(QObject):
     def __init__(self, parent=None):
         super().__init__(parent)
         self.captured_log = []
+        self._started = False
         self._invalid = []
         self._data = []
         self.proc = QProcess()
         self.proc.setReadChannel(QProcess.StandardError)
-        self.exit_expected = True  # Not started at all yet
+        self.exit_expected = None  # Not started at all yet
 
     def _log(self, line):
         """Add the given line to the captured log output."""
@@ -221,8 +223,8 @@ class Process(QObject):
 
     def start(self, args=None, *, env=None):
         """Start the process and wait until it started."""
-        self.exit_expected = False
         self._start(args, env=env)
+        self._started = True
         timeout = 60 if 'CI' in os.environ else 20
         for _ in range(timeout):
             with self._wait_signal(self.ready, timeout=1000,
@@ -230,6 +232,8 @@ class Process(QObject):
                 pass
 
             if not self.is_running():
+                if self.exit_expected:
+                    return
                 # _start ensures it actually started, but it might quit shortly
                 # afterwards
                 raise ProcessExited('\n' + _render_log(self.captured_log))
@@ -285,7 +289,7 @@ class Process(QObject):
             raise InvalidLine('\n' + '\n'.join(self._invalid))
 
         self.clear_data()
-        if not self.is_running() and not self.exit_expected:
+        if not self.is_running() and not self.exit_expected and self._started:
             raise ProcessExited
         self.exit_expected = False
 

tests/helpers/fixtures.py

@@ -279,7 +279,7 @@ def session_manager_stub(stubs):
 
 
 @pytest.fixture
-def tabbed_browser_stubs(stubs, win_registry):
+def tabbed_browser_stubs(qapp, stubs, win_registry):
     """Fixture providing a fake tabbed-browser object on win_id 0 and 1."""
     win_registry.add_window(1)
     stubs = [stubs.TabbedBrowserStub(), stubs.TabbedBrowserStub()]

tests/helpers/stubs.py

@@ -414,12 +414,24 @@ class FakeYamlConfig:
     """Fake configfiles.YamlConfig object."""
 
     def __init__(self):
-        self.values = {}
         self.loaded = False
+        self._values = {}
 
     def load(self):
         self.loaded = True
 
+    def __contains__(self, item):
+        return item in self._values
+
+    def __iter__(self):
+        return iter(self._values.items())
+
+    def __setitem__(self, key, value):
+        self._values[key] = value
+
+    def __getitem__(self, key):
+        return self._values[key]
+
 
 class StatusBarCommandStub(QLineEdit):
 

tests/helpers/test_stubs.py

@@ -36,8 +36,8 @@ def test_timeout(timer):
     func2 = mock.Mock()
     timer.timeout.connect(func)
     timer.timeout.connect(func2)
-    assert not func.called
+    func.assert_not_called()
-    assert not func2.called
+    func2.assert_not_called()
     timer.timeout.emit()
     func.assert_called_once_with()
     func2.assert_called_once_with()
@@ -49,7 +49,7 @@ def test_disconnect_all(timer):
     timer.timeout.connect(func)
     timer.timeout.disconnect()
     timer.timeout.emit()
-    assert not func.called
+    func.assert_not_called()
 
 
 def test_disconnect_one(timer):
@@ -58,7 +58,7 @@ def test_disconnect_one(timer):
     timer.timeout.connect(func)
     timer.timeout.disconnect(func)
     timer.timeout.emit()
-    assert not func.called
+    func.assert_not_called()
 
 
 def test_disconnect_all_invalid(timer):
@@ -74,8 +74,8 @@ def test_disconnect_one_invalid(timer):
     timer.timeout.connect(func1)
     with pytest.raises(TypeError):
         timer.timeout.disconnect(func2)
-    assert not func1.called
+    func1.assert_not_called()
-    assert not func2.called
+    func2.assert_not_called()
     timer.timeout.emit()
     func1.assert_called_once_with()
 

tests/unit/browser/test_history.py

@@ -127,21 +127,25 @@ def test_clear_force(qtbot, tmpdir, hist):
     assert not len(hist.completion)
 
 
-def test_delete_url(hist):
+@pytest.mark.parametrize('raw, escaped', [
+    ('http://example.com/1', 'http://example.com/1'),
+    ('http://example.com/1 2', 'http://example.com/1%202'),
+])
+def test_delete_url(hist, raw, escaped):
     hist.add_url(QUrl('http://example.com/'), atime=0)
-    hist.add_url(QUrl('http://example.com/1'), atime=0)
+    hist.add_url(QUrl(escaped), atime=0)
     hist.add_url(QUrl('http://example.com/2'), atime=0)
 
     before = set(hist)
     completion_before = set(hist.completion)
 
-    hist.delete_url(QUrl('http://example.com/1'))
+    hist.delete_url(QUrl(raw))
 
     diff = before.difference(set(hist))
-    assert diff == {('http://example.com/1', '', 0, False)}
+    assert diff == {(escaped, '', 0, False)}
 
     completion_diff = completion_before.difference(set(hist.completion))
-    assert completion_diff == {('http://example.com/1', '', 0)}
+    assert completion_diff == {(raw, '', 0)}
 
 
 @pytest.mark.parametrize(
@@ -280,6 +284,22 @@ def test_import_txt(hist, data_tmpdir, monkeypatch, stubs):
     assert (data_tmpdir / 'history.bak').exists()
 
 
+def test_import_txt_existing_backup(hist, data_tmpdir, monkeypatch, stubs):
+    monkeypatch.setattr(history, 'QTimer', stubs.InstaTimer)
+    histfile = data_tmpdir / 'history'
+    bakfile = data_tmpdir / 'history.bak'
+    histfile.write('12345 http://example.com/ title')
+    bakfile.write('12346 http://qutebrowser.org/')
+
+    hist.import_txt()
+
+    assert list(hist) == [('http://example.com/', 'title', 12345, False)]
+
+    assert not histfile.exists()
+    assert bakfile.read().split('\n') == ['12346 http://qutebrowser.org/',
+                                          '12345 http://example.com/ title']
+
+
 @pytest.mark.parametrize('line', [
     '',
     '#12345 http://example.com/commented',

tests/unit/browser/webkit/network/test_filescheme.py

@@ -26,7 +26,7 @@ from PyQt5.QtCore import QUrl
 from PyQt5.QtNetwork import QNetworkRequest
 
 from qutebrowser.browser.webkit.network import filescheme
-from qutebrowser.utils import urlutils
+from qutebrowser.utils import urlutils, utils
 
 
 @pytest.mark.parametrize('create_file, create_dir, filterfunc, expected', [
@@ -228,10 +228,7 @@ class TestDirbrowserHtml:
         assert parsed.folders == [bar_item]
 
     def test_root_dir(self, tmpdir, parser):
-        if os.name == 'nt':
+        root_dir = 'C:\\' if utils.is_windows else '/'
-            root_dir = 'C:\\'
-        else:
-            root_dir = '/'
         parsed = parser(root_dir)
         assert not parsed.parent
 

tests/unit/commands/test_runners.py

@@ -22,7 +22,6 @@
 import pytest
 
 from qutebrowser.commands import runners, cmdexc
-from qutebrowser.config import configtypes
 
 
 class TestCommandParser:
@@ -47,7 +46,6 @@ class TestCommandParser:
         if not cmdline_test.cmd:
             pytest.skip("Empty command")
 
-        monkeypatch.setattr(configtypes.Command, 'unvalidated', True)
         config_stub.val.aliases = {'alias_name': cmdline_test.cmd}
 
         parser = runners.CommandParser()

tests/unit/commands/test_userscripts.py

@@ -27,6 +27,7 @@ import pytest
 from PyQt5.QtCore import QFileSystemWatcher
 
 from qutebrowser.commands import userscripts
+from qutebrowser.utils import utils
 
 
 @pytest.mark.posix
@@ -60,7 +61,7 @@ class TestQtFIFOReader:
     userscripts._WindowsUserscriptRunner,
 ])
 def runner(request, runtime_tmpdir):
-    if (os.name != 'posix' and
+    if (not utils.is_posix and
             request.param is userscripts._POSIXUserscriptRunner):
         pytest.skip("Requires a POSIX os")
     else:
@@ -245,8 +246,8 @@ def test_unicode_error(caplog, qtbot, py_proc, runner):
     assert caplog.records[0].message == expected
 
 
-def test_unsupported(monkeypatch, tabbed_browser_stubs):
+@pytest.mark.fake_os('unknown')
-    monkeypatch.setattr(userscripts.os, 'name', 'toaster')
+def test_unsupported(tabbed_browser_stubs):
     with pytest.raises(userscripts.UnsupportedError, match="Userscripts are "
                        "not supported on this platform!"):
         userscripts.run_async(tab=None, cmd=None, win_id=0, env=None)

tests/unit/completion/test_completionmodel.py

@@ -103,7 +103,7 @@ def test_delete_cur_item_no_func():
     parent = model.index(0, 0)
     with pytest.raises(cmdexc.CommandError):
         model.delete_cur_item(model.index(0, 0, parent))
-    assert not callback.called
+    callback.assert_not_called()
 
 
 def test_delete_cur_item_no_cat():
@@ -114,4 +114,4 @@ def test_delete_cur_item_no_cat():
     model.rowsRemoved.connect(callback)
     with pytest.raises(qtutils.QtValueError):
         model.delete_cur_item(QModelIndex())
-    assert not callback.called
+    callback.assert_not_called()

tests/unit/completion/test_completionwidget.py

@@ -242,7 +242,7 @@ def test_completion_item_del_no_selection(completionview):
     completionview.set_model(model)
     with pytest.raises(cmdexc.CommandError, match='No item selected!'):
         completionview.completion_item_del()
-    assert not func.called
+    func.assert_not_called()
 
 
 def test_resize_no_model(completionview, qtbot):

tests/unit/completion/test_models.py

@@ -573,6 +573,24 @@ def test_bind_completion(qtmodeltester, cmdutils_stub, config_stub,
     })
 
 
+def test_bind_completion_no_current(qtmodeltester, cmdutils_stub, config_stub,
+                                    key_config_stub, configdata_stub, info):
+    """Test keybinding completion with no current binding."""
+    model = configmodel.bind('x', info=info)
+    model.set_pattern('')
+    qtmodeltester.data_display_may_return_none = True
+    qtmodeltester.check(model)
+
+    _check_completions(model, {
+        "Commands": [
+            ('open', 'open a url', ''),
+            ('q', "Alias for 'quit'", ''),
+            ('quit', 'quit qutebrowser', 'ZQ, <ctrl+q>'),
+            ('scroll', 'Scroll the current tab in the given direction.', '')
+        ],
+    })
+
+
 def test_url_completion_benchmark(benchmark, info,
                                   quickmark_manager_stub,
                                   bookmark_manager_stub,

tests/unit/config/test_config.py

@@ -18,19 +18,15 @@
 
 """Tests for qutebrowser.config.config."""
 
-import sys
 import copy
 import types
-import logging
-import unittest.mock
 
 import pytest
 from PyQt5.QtCore import QObject, QUrl
 from PyQt5.QtGui import QColor
 
-from qutebrowser import qutebrowser
 from qutebrowser.commands import cmdexc
-from qutebrowser.config import config, configdata, configexc, configfiles
+from qutebrowser.config import config, configdata, configexc
 from qutebrowser.utils import objreg, usertypes
 from qutebrowser.misc import objects
 
@@ -52,8 +48,8 @@ class TestChangeFilter:
 
     @pytest.fixture(autouse=True)
     def cleanup_globals(self, monkeypatch):
-        """Make sure config._change_filters is cleaned up."""
+        """Make sure config.change_filters is cleaned up."""
-        monkeypatch.setattr(config, '_change_filters', [])
+        monkeypatch.setattr(config, 'change_filters', [])
 
     @pytest.mark.parametrize('option', ['foobar', 'tab', 'tabss', 'tabs.'])
     def test_unknown_option(self, option):
@@ -65,7 +61,7 @@ class TestChangeFilter:
     def test_validate(self, option):
         cf = config.change_filter(option)
         cf.validate()
-        assert cf in config._change_filters
+        assert cf in config.change_filters
 
     @pytest.mark.parametrize('method', [True, False])
     @pytest.mark.parametrize('option, changed, matches', [
@@ -182,17 +178,6 @@ class TestKeyConfig:
         config_stub.val.bindings.commands = {'normal': bindings}
         assert keyconf.get_reverse_bindings_for('normal') == expected
 
-    def test_bind_invalid_command(self, keyconf):
-        with pytest.raises(configexc.KeybindingError,
-                           match='Invalid command: foobar'):
-            keyconf.bind('a', 'foobar', mode='normal')
-
-    def test_bind_invalid_mode(self, keyconf):
-        with pytest.raises(configexc.KeybindingError,
-                           match='completion-item-del: This command is only '
-                           'allowed in command mode, not normal.'):
-            keyconf.bind('a', 'completion-item-del', mode='normal')
-
     @pytest.mark.parametrize('force', [True, False])
     @pytest.mark.parametrize('key', ['a', '<Ctrl-X>', 'b'])
     def test_bind_duplicate(self, keyconf, config_stub, force, key):
@@ -208,12 +193,15 @@ class TestKeyConfig:
             assert keyconf.get_command(key, 'normal') == 'nop'
 
     @pytest.mark.parametrize('mode', ['normal', 'caret'])
-    def test_bind(self, keyconf, config_stub, qtbot, no_bindings, mode):
+    @pytest.mark.parametrize('command', [
+        'message-info foo',
+        'nop ;; wq',  # https://github.com/qutebrowser/qutebrowser/issues/3002
+    ])
+    def test_bind(self, keyconf, config_stub, qtbot, no_bindings,
+                  mode, command):
         config_stub.val.bindings.default = no_bindings
         config_stub.val.bindings.commands = no_bindings
 
-        command = 'message-info foo'
-
         with qtbot.wait_signal(config_stub.changed):
             keyconf.bind('a', command, mode=mode)
 
@@ -221,6 +209,16 @@ class TestKeyConfig:
         assert keyconf.get_bindings_for(mode)['a'] == command
         assert keyconf.get_command('a', mode) == command
 
+    def test_bind_mode_changing(self, keyconf, config_stub, no_bindings):
+        """Make sure we can bind to a command which changes the mode.
+
+        https://github.com/qutebrowser/qutebrowser/issues/2989
+        """
+        config_stub.val.bindings.default = no_bindings
+        config_stub.val.bindings.commands = no_bindings
+        keyconf.bind('a', 'set-cmd-text :nop ;; rl-beginning-of-line',
+                     mode='normal')
+
     @pytest.mark.parametrize('key, normalized', [
         ('a', 'a'),  # default bindings
         ('b', 'b'),  # custom bindings
@@ -317,9 +315,9 @@ class TestSetConfigCommand:
         assert config_stub.get(option) == new_value
 
         if temp:
-            assert option not in config_stub._yaml.values
+            assert option not in config_stub._yaml
         else:
-            assert config_stub._yaml.values[option] == new_value
+            assert config_stub._yaml[option] == new_value
 
     @pytest.mark.parametrize('temp', [True, False])
     def test_set_temp_override(self, commands, config_stub, temp):
@@ -335,7 +333,7 @@ class TestSetConfigCommand:
         commands.set(0, 'url.auto_search', 'never', temp=True)
 
         assert config_stub.val.url.auto_search == 'never'
-        assert config_stub._yaml.values['url.auto_search'] == 'dns'
+        assert config_stub._yaml['url.auto_search'] == 'dns'
 
     def test_set_print(self, config_stub, commands, message_mock):
         """Run ':set -p url.auto_search never'.
@@ -357,7 +355,7 @@ class TestSetConfigCommand:
         assert not config_stub.val.auto_save.session
         commands.set(0, 'auto_save.session!')
         assert config_stub.val.auto_save.session
-        assert config_stub._yaml.values['auto_save.session']
+        assert config_stub._yaml['auto_save.session']
 
     def test_set_toggle_nonbool(self, commands, config_stub):
         """Run ':set url.auto_search!'.
@@ -439,7 +437,19 @@ class TestSetConfigCommand:
         config_stub.set_obj(opt, initial)
         commands.set(0, opt, 'green', 'magenta', 'blue', 'yellow')
         assert config_stub.get(opt) == expected
-        assert config_stub._yaml.values[opt] == expected
+        assert config_stub._yaml[opt] == expected
+
+    def test_cycling_different_representation(self, commands, config_stub):
+        """When using a different representation, cycling should work.
+
+        For example, we use [foo] which is represented as ["foo"].
+        """
+        opt = 'qt_args'
+        config_stub.set_obj(opt, ['foo'])
+        commands.set(0, opt, '[foo]', '[bar]')
+        assert config_stub.get(opt) == ['bar']
+        commands.set(0, opt, '[foo]', '[bar]')
+        assert config_stub.get(opt) == ['foo']
 
 
 class TestBindConfigCommand:
@@ -464,7 +474,7 @@ class TestBindConfigCommand:
 
         commands.bind('a', command)
         assert keyconf.get_command('a', 'normal') == command
-        yaml_bindings = config_stub._yaml.values['bindings.commands']['normal']
+        yaml_bindings = config_stub._yaml['bindings.commands']['normal']
         assert yaml_bindings['a'] == command
 
     @pytest.mark.parametrize('key, mode, expected', [
@@ -504,20 +514,14 @@ class TestBindConfigCommand:
         msg = message_mock.getmsg(usertypes.MessageLevel.info)
         assert msg.text == expected
 
-    @pytest.mark.parametrize('command, mode, expected', [
+    def test_bind_invalid_mode(self, commands):
-        ('foobar', 'normal', "bind: Invalid command: foobar"),
+        """Run ':bind --mode=wrongmode nop'.
-        ('completion-item-del', 'normal',
-            "bind: completion-item-del: This command is only allowed in "
-            "command mode, not normal."),
-        ('nop', 'wrongmode', "bind: Invalid mode wrongmode!"),
-    ])
-    def test_bind_invalid(self, commands, command, mode, expected):
-        """Run ':bind a foobar' / ':bind a completion-item-del'.
 
         Should show an error.
         """
-        with pytest.raises(cmdexc.CommandError, match=expected):
+        with pytest.raises(cmdexc.CommandError,
-            commands.bind('a', command, mode=mode)
+                           match='bind: Invalid mode wrongmode!'):
+            commands.bind('a', 'nop', mode='wrongmode')
 
     @pytest.mark.parametrize('force', [True, False])
     @pytest.mark.parametrize('key', ['a', 'b', '<Ctrl-X>'])
@@ -565,7 +569,7 @@ class TestBindConfigCommand:
         commands.unbind(key)
         assert keyconf.get_command(key, 'normal') is None
 
-        yaml_bindings = config_stub._yaml.values['bindings.commands']['normal']
+        yaml_bindings = config_stub._yaml['bindings.commands']['normal']
         if key in 'bc':
             # Custom binding
             assert normalized not in yaml_bindings
@@ -612,18 +616,13 @@ class TestConfig:
 
     def test_read_yaml(self, conf):
         assert not conf._yaml.loaded
-        conf._yaml.values['content.plugins'] = True
+        conf._yaml['content.plugins'] = True
 
         conf.read_yaml()
 
         assert conf._yaml.loaded
         assert conf._values['content.plugins'] is True
 
-    def test_read_yaml_invalid(self, conf):
-        conf._yaml.values['foo.bar'] = True
-        with pytest.raises(configexc.NoOptionError):
-            conf.read_yaml()
-
     def test_get_opt_valid(self, conf):
         assert conf.get_opt('tabs.show') == configdata.DATA['tabs.show']
 
@@ -743,9 +742,9 @@ class TestConfig:
             meth(option, value, save_yaml=save_yaml)
         assert conf._values[option] is True
         if save_yaml:
-            assert conf._yaml.values[option] is True
+            assert conf._yaml[option] is True
         else:
-            assert option not in conf._yaml.values
+            assert option not in conf._yaml
 
     @pytest.mark.parametrize('method', ['set_obj', 'set_str'])
     def test_set_invalid(self, conf, qtbot, method):
@@ -873,205 +872,3 @@ def test_set_register_stylesheet(delete, stylesheet_param, update, qtbot,
         expected = 'yellow'
 
     assert obj.rendered_stylesheet == expected
-
-
-@pytest.fixture
-def init_patch(qapp, fake_save_manager, monkeypatch, config_tmpdir,
-               data_tmpdir):
-    monkeypatch.setattr(configdata, 'DATA', None)
-    monkeypatch.setattr(configfiles, 'state', None)
-    monkeypatch.setattr(config, 'instance', None)
-    monkeypatch.setattr(config, 'key_instance', None)
-    monkeypatch.setattr(config, '_change_filters', [])
-    monkeypatch.setattr(config, '_init_errors', [])
-    # Make sure we get no SSL warning
-    monkeypatch.setattr(config.earlyinit, 'check_backend_ssl_support',
-                        lambda _backend: None)
-    yield
-    try:
-        objreg.delete('config-commands')
-    except KeyError:
-        pass
-
-
-@pytest.mark.parametrize('load_autoconfig', [True, False])  # noqa
-@pytest.mark.parametrize('config_py', [True, 'error', False])
-@pytest.mark.parametrize('invalid_yaml', ['42', 'unknown', False])
-# pylint: disable=too-many-branches
-def test_early_init(init_patch, config_tmpdir, caplog, fake_args,
-                    load_autoconfig, config_py, invalid_yaml):
-    # Prepare files
-    autoconfig_file = config_tmpdir / 'autoconfig.yml'
-    config_py_file = config_tmpdir / 'config.py'
-
-    if invalid_yaml == '42':
-        autoconfig_file.write_text('42', 'utf-8', ensure=True)
-    elif invalid_yaml == 'unknown':
-        autoconfig_file.write_text('global:\n  colors.foobar: magenta\n',
-                                   'utf-8', ensure=True)
-    else:
-        assert not invalid_yaml
-        autoconfig_file.write_text('global:\n  colors.hints.fg: magenta\n',
-                                   'utf-8', ensure=True)
-
-    if config_py:
-        config_py_lines = ['c.colors.hints.bg = "red"']
-        if not load_autoconfig:
-            config_py_lines.append('config.load_autoconfig = False')
-        if config_py == 'error':
-            config_py_lines.append('c.foo = 42')
-        config_py_file.write_text('\n'.join(config_py_lines),
-                                  'utf-8', ensure=True)
-
-    with caplog.at_level(logging.ERROR):
-        config.early_init(fake_args)
-
-    # Check error messages
-    expected_errors = []
-    if config_py == 'error':
-        expected_errors.append(
-            "Errors occurred while reading config.py:\n"
-            "  While setting 'foo': No option 'foo'")
-    if invalid_yaml and (load_autoconfig or not config_py):
-        error = "Errors occurred while reading autoconfig.yml:\n"
-        if invalid_yaml == '42':
-            error += "  While loading data: Toplevel object is not a dict"
-        elif invalid_yaml == 'unknown':
-            error += "  Error: No option 'colors.foobar'"
-        else:
-            assert False, invalid_yaml
-        expected_errors.append(error)
-
-    actual_errors = [str(err) for err in config._init_errors]
-    assert actual_errors == expected_errors
-
-    # Make sure things have been init'ed
-    objreg.get('config-commands')
-    assert isinstance(config.instance, config.Config)
-    assert isinstance(config.key_instance, config.KeyConfig)
-
-    # Check config values
-    if config_py and load_autoconfig and not invalid_yaml:
-        assert config.instance._values == {
-            'colors.hints.bg': 'red',
-            'colors.hints.fg': 'magenta',
-        }
-    elif config_py:
-        assert config.instance._values == {'colors.hints.bg': 'red'}
-    elif invalid_yaml:
-        assert config.instance._values == {}
-    else:
-        assert config.instance._values == {'colors.hints.fg': 'magenta'}
-
-
-def test_early_init_invalid_change_filter(init_patch, fake_args):
-    config.change_filter('foobar')
-    with pytest.raises(configexc.NoOptionError):
-        config.early_init(fake_args)
-
-
-@pytest.mark.parametrize('errors', [True, False])
-def test_late_init(init_patch, monkeypatch, fake_save_manager, fake_args,
-                   mocker, errors):
-    config.early_init(fake_args)
-    if errors:
-        err = configexc.ConfigErrorDesc("Error text", Exception("Exception"))
-        errs = configexc.ConfigFileErrors("config.py", [err])
-        monkeypatch.setattr(config, '_init_errors', [errs])
-    msgbox_mock = mocker.patch('qutebrowser.config.config.msgbox.msgbox',
-                               autospec=True)
-
-    config.late_init(fake_save_manager)
-
-    fake_save_manager.add_saveable.assert_any_call(
-        'state-config', unittest.mock.ANY)
-    fake_save_manager.add_saveable.assert_any_call(
-        'yaml-config', unittest.mock.ANY)
-    if errors:
-        assert len(msgbox_mock.call_args_list) == 1
-        _call_posargs, call_kwargs = msgbox_mock.call_args_list[0]
-        text = call_kwargs['text'].strip()
-        assert text.startswith('Errors occurred while reading config.py:')
-        assert '<b>Error text</b>: Exception' in text
-    else:
-        assert not msgbox_mock.called
-
-
-class TestQtArgs:
-
-    @pytest.fixture
-    def parser(self, mocker):
-        """Fixture to provide an argparser.
-
-        Monkey-patches .exit() of the argparser so it doesn't exit on errors.
-        """
-        parser = qutebrowser.get_argparser()
-        mocker.patch.object(parser, 'exit', side_effect=Exception)
-        return parser
-
-    @pytest.mark.parametrize('args, expected', [
-        # No Qt arguments
-        (['--debug'], [sys.argv[0]]),
-        # Qt flag
-        (['--debug', '--qt-flag', 'reverse'], [sys.argv[0], '--reverse']),
-        # Qt argument with value
-        (['--qt-arg', 'stylesheet', 'foo'],
-         [sys.argv[0], '--stylesheet', 'foo']),
-        # --qt-arg given twice
-        (['--qt-arg', 'stylesheet', 'foo', '--qt-arg', 'geometry', 'bar'],
-         [sys.argv[0], '--stylesheet', 'foo', '--geometry', 'bar']),
-        # --qt-flag given twice
-        (['--qt-flag', 'foo', '--qt-flag', 'bar'],
-         [sys.argv[0], '--foo', '--bar']),
-    ])
-    def test_qt_args(self, config_stub, args, expected, parser):
-        """Test commandline with no Qt arguments given."""
-        parsed = parser.parse_args(args)
-        assert config.qt_args(parsed) == expected
-
-    def test_qt_both(self, config_stub, parser):
-        """Test commandline with a Qt argument and flag."""
-        args = parser.parse_args(['--qt-arg', 'stylesheet', 'foobar',
-                                  '--qt-flag', 'reverse'])
-        qt_args = config.qt_args(args)
-        assert qt_args[0] == sys.argv[0]
-        assert '--reverse' in qt_args
-        assert '--stylesheet' in qt_args
-        assert 'foobar' in qt_args
-
-    def test_with_settings(self, config_stub, parser):
-        parsed = parser.parse_args(['--qt-flag', 'foo'])
-        config_stub.val.qt_args = ['bar']
-        assert config.qt_args(parsed) == [sys.argv[0], '--foo', '--bar']
-
-
-@pytest.mark.parametrize('arg, confval, can_import, is_new_webkit, used', [
-    # overridden by commandline arg
-    ('webkit', 'auto', False, False, usertypes.Backend.QtWebKit),
-    # overridden by config
-    (None, 'webkit', False, False, usertypes.Backend.QtWebKit),
-    # WebKit available but too old
-    (None, 'auto', True, False, usertypes.Backend.QtWebEngine),
-    # WebKit available and new
-    (None, 'auto', True, True, usertypes.Backend.QtWebKit),
-    # WebKit unavailable
-    (None, 'auto', False, False, usertypes.Backend.QtWebEngine),
-])
-def test_get_backend(monkeypatch, fake_args, config_stub,
-                     arg, confval, can_import, is_new_webkit, used):
-    real_import = __import__
-
-    def fake_import(name, *args, **kwargs):
-        if name != 'PyQt5.QtWebKit':
-            return real_import(name, *args, **kwargs)
-        if can_import:
-            return None
-        raise ImportError
-
-    fake_args.backend = arg
-    config_stub.val.backend = confval
-    monkeypatch.setattr(config.qtutils, 'is_new_qtwebkit',
-                        lambda: is_new_webkit)
-    monkeypatch.setattr('builtins.__import__', fake_import)
-
-    assert config.get_backend(fake_args) == used

tests/unit/config/test_configexc.py

@@ -49,6 +49,13 @@ def test_duplicate_key_error():
     assert str(e) == "Duplicate key asdf"
 
 
+def test_desc_with_text():
+    """Test ConfigErrorDesc.with_text."""
+    old = configexc.ConfigErrorDesc("Error text", Exception("Exception text"))
+    new = old.with_text("additional text")
+    assert str(new) == 'Error text (additional text): Exception text'
+
+
 @pytest.fixture
 def errors():
     """Get a ConfigFileErrors object."""

tests/unit/config/test_configfiles.py

@@ -23,11 +23,19 @@ import sys
 
 import pytest
 
-from qutebrowser.config import config, configfiles, configexc
+from qutebrowser.config import config, configfiles, configexc, configdata
+from qutebrowser.utils import utils
 
 from PyQt5.QtCore import QSettings
 
 
+@pytest.fixture(autouse=True)
+def configdata_init():
+    """Initialize configdata if needed."""
+    if configdata.DATA is None:
+        configdata.init()
+
+
 @pytest.mark.parametrize('old_data, insert, new_data', [
     (None, False, '[general]\n\n[geometry]\n\n'),
     ('[general]\nfooled = true', False, '[general]\n\n[geometry]\n\n'),
@@ -54,12 +62,16 @@ def test_state_config(fake_save_manager, data_tmpdir,
     assert statefile.read_text('utf-8') == new_data
 
 
-@pytest.mark.parametrize('old_config', [
+class TestYaml:
+
+    pytestmark = pytest.mark.usefixtures('fake_save_manager')
+
+    @pytest.mark.parametrize('old_config', [
         None,
         'global:\n  colors.hints.fg: magenta',
-])
+    ])
-@pytest.mark.parametrize('insert', [True, False])
+    @pytest.mark.parametrize('insert', [True, False])
-def test_yaml_config(fake_save_manager, config_tmpdir, old_config, insert):
+    def test_yaml_config(self, config_tmpdir, old_config, insert):
         autoconfig = config_tmpdir / 'autoconfig.yml'
         if old_config is not None:
             autoconfig.write_text(old_config, 'utf-8')
@@ -68,38 +80,103 @@ def test_yaml_config(fake_save_manager, config_tmpdir, old_config, insert):
         yaml.load()
 
         if insert:
-        yaml.values['tabs.show'] = 'never'
+            yaml['tabs.show'] = 'never'
 
         yaml._save()
 
+        if not insert and old_config is None:
+            lines = []
+        else:
             text = autoconfig.read_text('utf-8')
             lines = text.splitlines()
-    print(lines)
 
+            if insert:
                 assert lines[0].startswith('# DO NOT edit this file by hand,')
                 assert 'config_version: {}'.format(yaml.VERSION) in lines
 
-    if old_config is None and not insert:
-        assert 'global: {}' in lines
-    else:
             assert 'global:' in lines
 
+        print(lines)
+
         # WORKAROUND for https://github.com/PyCQA/pylint/issues/574
-    if 'magenta' in (old_config or ''):  # pylint: disable=superfluous-parens
+        # pylint: disable=superfluous-parens
+        if 'magenta' in (old_config or ''):
             assert '  colors.hints.fg: magenta' in lines
         if insert:
             assert '  tabs.show: never' in lines
 
+    def test_unknown_key(self, config_tmpdir):
+        """An unknown setting should be deleted."""
+        autoconfig = config_tmpdir / 'autoconfig.yml'
+        autoconfig.write_text('global:\n  hello: world', encoding='utf-8')
 
-@pytest.mark.parametrize('line, text, exception', [
+        yaml = configfiles.YamlConfig()
+        yaml.load()
+        yaml._save()
+
+        lines = autoconfig.read_text('utf-8').splitlines()
+        assert '  hello:' not in lines
+
+    @pytest.mark.parametrize('old_config', [
+        None,
+        'global:\n  colors.hints.fg: magenta',
+    ])
+    @pytest.mark.parametrize('key, value', [
+        ('colors.hints.fg', 'green'),
+        ('colors.hints.bg', None),
+        ('confirm_quit', True),
+        ('confirm_quit', False),
+    ])
+    def test_changed(self, qtbot, config_tmpdir, old_config, key, value):
+        autoconfig = config_tmpdir / 'autoconfig.yml'
+        if old_config is not None:
+            autoconfig.write_text(old_config, 'utf-8')
+
+        yaml = configfiles.YamlConfig()
+        yaml.load()
+
+        with qtbot.wait_signal(yaml.changed):
+            yaml[key] = value
+
+        assert key in yaml
+        assert yaml[key] == value
+
+        yaml._save()
+
+        yaml = configfiles.YamlConfig()
+        yaml.load()
+
+        assert key in yaml
+        assert yaml[key] == value
+
+    @pytest.mark.parametrize('old_config', [
+        None,
+        'global:\n  colors.hints.fg: magenta',
+    ])
+    def test_unchanged(self, config_tmpdir, old_config):
+        autoconfig = config_tmpdir / 'autoconfig.yml'
+        mtime = None
+        if old_config is not None:
+            autoconfig.write_text(old_config, 'utf-8')
+            mtime = autoconfig.stat().mtime
+
+        yaml = configfiles.YamlConfig()
+        yaml.load()
+        yaml._save()
+
+        if old_config is None:
+            assert not autoconfig.exists()
+        else:
+            assert autoconfig.stat().mtime == mtime
+
+    @pytest.mark.parametrize('line, text, exception', [
         ('%', 'While parsing', 'while scanning a directive'),
         ('global: 42', 'While loading data', "'global' object is not a dict"),
         ('foo: 42', 'While loading data',
         "Toplevel object does not contain 'global' key"),
         ('42', 'While loading data', "Toplevel object is not a dict"),
-])
+    ])
-def test_yaml_config_invalid(fake_save_manager, config_tmpdir,
+    def test_invalid(self, config_tmpdir, line, text, exception):
-                             line, text, exception):
         autoconfig = config_tmpdir / 'autoconfig.yml'
         autoconfig.write_text(line, 'utf-8', ensure=True)
 
@@ -114,8 +191,7 @@ def test_yaml_config_invalid(fake_save_manager, config_tmpdir,
         assert str(error.exception).splitlines()[0] == exception
         assert error.traceback is None
 
-
+    def test_oserror(self, config_tmpdir):
-def test_yaml_oserror(fake_save_manager, config_tmpdir):
         autoconfig = config_tmpdir / 'autoconfig.yml'
         autoconfig.ensure()
         autoconfig.chmod(0)
@@ -134,32 +210,121 @@ def test_yaml_oserror(fake_save_manager, config_tmpdir):
         assert error.traceback is None
 
 
+class ConfPy:
+
+    """Helper class to get a confpy fixture."""
+
+    def __init__(self, tmpdir, filename: str = "config.py"):
+        self._file = tmpdir / filename
+        self.filename = str(self._file)
+
+    def write(self, *lines):
+        text = '\n'.join(lines)
+        self._file.write_text(text, 'utf-8', ensure=True)
+
+    def read(self, error=False):
+        """Read the config.py via configfiles and check for errors."""
+        if error:
+            with pytest.raises(configexc.ConfigFileErrors) as excinfo:
+                configfiles.read_config_py(self.filename)
+            errors = excinfo.value.errors
+            assert len(errors) == 1
+            return errors[0]
+        else:
+            configfiles.read_config_py(self.filename, raising=True)
+            return None
+
+    def write_qbmodule(self):
+        self.write('import qbmodule',
+                   'qbmodule.run(config)')
+
+
+class TestConfigPyModules:
+
+    pytestmark = pytest.mark.usefixtures('config_stub', 'key_config_stub')
+
+    @pytest.fixture
+    def confpy(self, tmpdir, config_tmpdir, data_tmpdir):
+        return ConfPy(tmpdir)
+
+    @pytest.fixture
+    def qbmodulepy(self, tmpdir):
+        return ConfPy(tmpdir, filename="qbmodule.py")
+
+    @pytest.fixture(autouse=True)
+    def restore_sys_path(self):
+        old_path = sys.path.copy()
+        yield
+        sys.path = old_path
+
+    def test_bind_in_module(self, confpy, qbmodulepy, tmpdir):
+        qbmodulepy.write('def run(config):',
+        '    config.bind(",a", "message-info foo", mode="normal")')
+        confpy.write_qbmodule()
+        confpy.read()
+        expected = {'normal': {',a': 'message-info foo'}}
+        assert config.instance._values['bindings.commands'] == expected
+        assert "qbmodule" not in sys.modules.keys()
+        assert tmpdir not in sys.path
+
+    def test_restore_sys_on_err(self, confpy, qbmodulepy, tmpdir):
+        confpy.write_qbmodule()
+        qbmodulepy.write('def run(config):',
+                         '    1/0')
+        error = confpy.read(error=True)
+
+        assert error.text == "Unhandled exception"
+        assert isinstance(error.exception, ZeroDivisionError)
+        assert "qbmodule" not in sys.modules.keys()
+        assert tmpdir not in sys.path
+
+    def test_fail_on_nonexistent_module(self, confpy, qbmodulepy, tmpdir):
+        qbmodulepy.write('def run(config):',
+                         '    pass')
+        confpy.write('import foobar',
+                     'foobar.run(config)')
+
+        error = confpy.read(error=True)
+
+        assert error.text == "Unhandled exception"
+        assert isinstance(error.exception, ImportError)
+
+        tblines = error.traceback.strip().splitlines()
+        assert tblines[0] == "Traceback (most recent call last):"
+        assert tblines[-1].endswith("Error: No module named 'foobar'")
+
+    def test_no_double_if_path_exists(self, confpy, qbmodulepy, tmpdir):
+        sys.path.insert(0, tmpdir)
+        confpy.write('import sys',
+                     'if sys.path[0] in sys.path[1:]:',
+                     '    raise Exception("Path not expected")')
+        confpy.read()
+        assert sys.path.count(tmpdir) == 1
+
+
 class TestConfigPy:
 
     """Tests for ConfigAPI and read_config_py()."""
 
     pytestmark = pytest.mark.usefixtures('config_stub', 'key_config_stub')
 
-    class ConfPy:
-
-        """Helper class to get a confpy fixture."""
-
-        def __init__(self, tmpdir):
-            self._confpy = tmpdir / 'config.py'
-            self.filename = str(self._confpy)
-
-        def write(self, *lines):
-            text = '\n'.join(lines)
-            self._confpy.write_text(text, 'utf-8', ensure=True)
-
-        def read(self):
-            """Read the config.py via configfiles and check for errors."""
-            api = configfiles.read_config_py(self.filename)
-            assert not api.errors
-
     @pytest.fixture
-    def confpy(self, tmpdir):
+    def confpy(self, tmpdir, config_tmpdir, data_tmpdir):
-        return self.ConfPy(tmpdir)
+        return ConfPy(tmpdir)
+
+    def test_assertions(self, confpy):
+        """Make sure assertions in config.py work for these tests."""
+        confpy.write('assert False')
+        with pytest.raises(AssertionError):
+            confpy.read()  # no errors=True so it gets raised
+
+    @pytest.mark.parametrize('what', ['configdir', 'datadir'])
+    def test_getting_dirs(self, confpy, what):
+        confpy.write('import pathlib',
+                     'directory = config.{}'.format(what),
+                     'assert isinstance(directory, pathlib.Path)',
+                     'assert directory.exists()')
+        confpy.read()
 
     @pytest.mark.parametrize('line', [
         'c.colors.hints.bg = "red"',
@@ -176,25 +341,15 @@ class TestConfigPy:
         'config.get("colors.hints.fg")',
     ])
     def test_get(self, confpy, set_first, get_line):
-        """Test whether getting options works correctly.
+        """Test whether getting options works correctly."""
-
-        We test this by doing the following:
-           - Set colors.hints.fg to some value (inside the config.py with
-             set_first, outside of it otherwise).
-           - In the config.py, read .fg and set .bg to the same value.
-           - Verify that .bg has been set correctly.
-        """
         # pylint: disable=bad-config-option
         config.val.colors.hints.fg = 'green'
         if set_first:
             confpy.write('c.colors.hints.fg = "red"',
-                         'c.colors.hints.bg = {}'.format(get_line))
+                         'assert {} == "red"'.format(get_line))
-            expected = 'red'
         else:
-            confpy.write('c.colors.hints.bg = {}'.format(get_line))
+            confpy.write('assert {} == "green"'.format(get_line))
-            expected = 'green'
         confpy.read()
-        assert config.instance._values['colors.hints.bg'] == expected
 
     @pytest.mark.parametrize('line, mode', [
         ('config.bind(",a", "message-info foo")', 'normal'),
@@ -206,6 +361,23 @@ class TestConfigPy:
         expected = {mode: {',a': 'message-info foo'}}
         assert config.instance._values['bindings.commands'] == expected
 
+    def test_bind_freshly_defined_alias(self, confpy):
+        """Make sure we can bind to a new alias.
+
+        https://github.com/qutebrowser/qutebrowser/issues/3001
+        """
+        confpy.write("c.aliases['foo'] = 'message-info foo'",
+                     "config.bind(',f', 'foo')")
+        confpy.read()
+
+    def test_bind_duplicate_key(self, confpy):
+        """Make sure we get a nice error message on duplicate key bindings."""
+        confpy.write("config.bind('H', 'message-info back')")
+        error = confpy.read(error=True)
+
+        expected = "Duplicate key H - use force=True to override!"
+        assert str(error.exception) == expected
+
     @pytest.mark.parametrize('line, key, mode', [
         ('config.unbind("o")', 'o', 'normal'),
         ('config.unbind("y", mode="prompt")', 'y', 'prompt'),
@@ -223,17 +395,7 @@ class TestConfigPy:
         assert config.instance._values['aliases']['foo'] == 'message-info foo'
         assert config.instance._values['aliases']['bar'] == 'message-info bar'
 
-    def test_reading_default_location(self, config_tmpdir):
+    def test_oserror(self, tmpdir, data_tmpdir, config_tmpdir):
-        (config_tmpdir / 'config.py').write_text(
-            'c.colors.hints.bg = "red"', 'utf-8')
-        configfiles.read_config_py()
-        assert config.instance._values['colors.hints.bg'] == 'red'
-
-    def test_reading_missing_default_location(self, config_tmpdir):
-        assert not (config_tmpdir / 'config.py').exists()
-        configfiles.read_config_py()  # Should not crash
-
-    def test_oserror(self, tmpdir):
         with pytest.raises(configexc.ConfigFileErrors) as excinfo:
             configfiles.read_config_py(str(tmpdir / 'foo'))
 
@@ -250,7 +412,7 @@ class TestConfigPy:
 
         assert len(excinfo.value.errors) == 1
         error = excinfo.value.errors[0]
-        assert isinstance(error.exception, (TypeError, ValueError))
+        assert isinstance(error.exception, ValueError)
         assert error.text == "Error while compiling"
         exception_text = 'source code string cannot contain null bytes'
         assert str(error.exception) == exception_text
@@ -275,13 +437,9 @@ class TestConfigPy:
         assert "    ^" in tblines
 
     def test_unhandled_exception(self, confpy):
-        confpy.write("config.load_autoconfig = False", "1/0")
+        confpy.write("1/0")
-        api = configfiles.read_config_py(confpy.filename)
+        error = confpy.read(error=True)
 
-        assert not api.load_autoconfig
-
-        assert len(api.errors) == 1
-        error = api.errors[0]
         assert error.text == "Unhandled exception"
         assert isinstance(error.exception, ZeroDivisionError)
 
@@ -293,9 +451,8 @@ class TestConfigPy:
     def test_config_val(self, confpy):
         """Using config.val should not work in config.py files."""
         confpy.write("config.val.colors.hints.bg = 'red'")
-        api = configfiles.read_config_py(confpy.filename)
+        error = confpy.read(error=True)
-        assert len(api.errors) == 1
+
-        error = api.errors[0]
         assert error.text == "Unhandled exception"
         assert isinstance(error.exception, AttributeError)
         message = "'ConfigAPI' object has no attribute 'val'"
@@ -303,13 +460,9 @@ class TestConfigPy:
 
     @pytest.mark.parametrize('line', ["c.foo = 42", "config.set('foo', 42)"])
     def test_config_error(self, confpy, line):
-        confpy.write(line, "config.load_autoconfig = False")
+        confpy.write(line)
-        api = configfiles.read_config_py(confpy.filename)
+        error = confpy.read(error=True)
 
-        assert not api.load_autoconfig
-
-        assert len(api.errors) == 1
-        error = api.errors[0]
         assert error.text == "While setting 'foo'"
         assert isinstance(error.exception, configexc.NoOptionError)
         assert str(error.exception) == "No option 'foo'"
@@ -317,16 +470,20 @@ class TestConfigPy:
 
     def test_multiple_errors(self, confpy):
         confpy.write("c.foo = 42", "config.set('foo', 42)", "1/0")
-        api = configfiles.read_config_py(confpy.filename)
-        assert len(api.errors) == 3
 
-        for error in api.errors[:2]:
+        with pytest.raises(configexc.ConfigFileErrors) as excinfo:
+            configfiles.read_config_py(confpy.filename)
+
+        errors = excinfo.value.errors
+        assert len(errors) == 3
+
+        for error in errors[:2]:
             assert error.text == "While setting 'foo'"
             assert isinstance(error.exception, configexc.NoOptionError)
             assert str(error.exception) == "No option 'foo'"
             assert error.traceback is None
 
-        error = api.errors[2]
+        error = errors[2]
         assert error.text == "Unhandled exception"
         assert isinstance(error.exception, ZeroDivisionError)
         assert error.traceback is not None
@@ -343,7 +500,7 @@ def test_init(init_patch, config_tmpdir):
     configfiles.init()
 
     # Make sure qsettings land in a subdir
-    if sys.platform == 'linux':
+    if utils.is_linux:
         settings = QSettings()
         settings.setValue("hello", "world")
         settings.sync()

tests/unit/config/test_configinit.py

@@ -0,0 +1,274 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+# Copyright 2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Tests for qutebrowser.config.configinit."""
+
+import sys
+import logging
+import unittest.mock
+
+import pytest
+
+from qutebrowser import qutebrowser
+from qutebrowser.config import (config, configdata, configexc, configfiles,
+                                configinit)
+from qutebrowser.utils import objreg, usertypes
+
+
+@pytest.fixture
+def init_patch(qapp, fake_save_manager, monkeypatch, config_tmpdir,
+               data_tmpdir):
+    monkeypatch.setattr(configdata, 'DATA', None)
+    monkeypatch.setattr(configfiles, 'state', None)
+    monkeypatch.setattr(config, 'instance', None)
+    monkeypatch.setattr(config, 'key_instance', None)
+    monkeypatch.setattr(config, 'change_filters', [])
+    monkeypatch.setattr(configinit, '_init_errors', None)
+    # Make sure we get no SSL warning
+    monkeypatch.setattr(configinit.earlyinit, 'check_backend_ssl_support',
+                        lambda _backend: None)
+    yield
+    try:
+        objreg.delete('config-commands')
+    except KeyError:
+        pass
+
+
+class TestEarlyInit:
+
+    @pytest.mark.parametrize('config_py', [True, 'error', False])
+    def test_config_py(self, init_patch, config_tmpdir, caplog, fake_args,
+                       config_py):
+        """Test loading with only a config.py."""
+        config_py_file = config_tmpdir / 'config.py'
+
+        if config_py:
+            config_py_lines = ['c.colors.hints.bg = "red"']
+            if config_py == 'error':
+                config_py_lines.append('c.foo = 42')
+            config_py_file.write_text('\n'.join(config_py_lines),
+                                      'utf-8', ensure=True)
+
+        with caplog.at_level(logging.ERROR):
+            configinit.early_init(fake_args)
+
+        # Check error messages
+        expected_errors = []
+        if config_py == 'error':
+            expected_errors.append("While setting 'foo': No option 'foo'")
+
+        if configinit._init_errors is None:
+            actual_errors = []
+        else:
+            actual_errors = [str(err)
+                             for err in configinit._init_errors.errors]
+
+        assert actual_errors == expected_errors
+
+        # Make sure things have been init'ed
+        objreg.get('config-commands')
+        assert isinstance(config.instance, config.Config)
+        assert isinstance(config.key_instance, config.KeyConfig)
+
+        # Check config values
+        if config_py:
+            assert config.instance._values == {'colors.hints.bg': 'red'}
+        else:
+            assert config.instance._values == {}
+
+    @pytest.mark.parametrize('load_autoconfig', [True, False])
+    @pytest.mark.parametrize('config_py', [True, 'error', False])
+    @pytest.mark.parametrize('invalid_yaml', ['42', 'unknown', 'wrong-type',
+                                              False])
+    def test_autoconfig_yml(self, init_patch, config_tmpdir, caplog, fake_args,
+                            load_autoconfig, config_py, invalid_yaml):
+        """Test interaction between config.py and autoconfig.yml."""
+        # pylint: disable=too-many-locals,too-many-branches
+        # Prepare files
+        autoconfig_file = config_tmpdir / 'autoconfig.yml'
+        config_py_file = config_tmpdir / 'config.py'
+
+        yaml_text = {
+            '42': '42',
+            'unknown': 'global:\n  colors.foobar: magenta\n',
+            'wrong-type': 'global:\n  tabs.position: true\n',
+            False: 'global:\n  colors.hints.fg: magenta\n',
+        }
+        autoconfig_file.write_text(yaml_text[invalid_yaml], 'utf-8',
+                                   ensure=True)
+
+        if config_py:
+            config_py_lines = ['c.colors.hints.bg = "red"']
+            if load_autoconfig:
+                config_py_lines.append('config.load_autoconfig()')
+            if config_py == 'error':
+                config_py_lines.append('c.foo = 42')
+            config_py_file.write_text('\n'.join(config_py_lines),
+                                      'utf-8', ensure=True)
+
+        with caplog.at_level(logging.ERROR):
+            configinit.early_init(fake_args)
+
+        # Check error messages
+        expected_errors = []
+
+        if load_autoconfig or not config_py:
+            suffix = ' (autoconfig.yml)' if config_py else ''
+            if invalid_yaml == '42':
+                error = ("While loading data{}: Toplevel object is not a dict"
+                        .format(suffix))
+                expected_errors.append(error)
+            elif invalid_yaml == 'wrong-type':
+                error = ("Error{}: Invalid value 'True' - expected a value of "
+                        "type str but got bool.".format(suffix))
+                expected_errors.append(error)
+        if config_py == 'error':
+            expected_errors.append("While setting 'foo': No option 'foo'")
+
+        if configinit._init_errors is None:
+            actual_errors = []
+        else:
+            actual_errors = [str(err)
+                             for err in configinit._init_errors.errors]
+
+        assert actual_errors == expected_errors
+
+        # Check config values
+        if config_py and load_autoconfig and not invalid_yaml:
+            assert config.instance._values == {
+                'colors.hints.bg': 'red',
+                'colors.hints.fg': 'magenta',
+            }
+        elif config_py:
+            assert config.instance._values == {'colors.hints.bg': 'red'}
+        elif invalid_yaml:
+            assert config.instance._values == {}
+        else:
+            assert config.instance._values == {'colors.hints.fg': 'magenta'}
+
+    def test_invalid_change_filter(self, init_patch, fake_args):
+        config.change_filter('foobar')
+        with pytest.raises(configexc.NoOptionError):
+            configinit.early_init(fake_args)
+
+
+@pytest.mark.parametrize('errors', [True, False])
+def test_late_init(init_patch, monkeypatch, fake_save_manager, fake_args,
+                   mocker, errors):
+    configinit.early_init(fake_args)
+    if errors:
+        err = configexc.ConfigErrorDesc("Error text", Exception("Exception"))
+        errs = configexc.ConfigFileErrors("config.py", [err])
+        monkeypatch.setattr(configinit, '_init_errors', errs)
+    msgbox_mock = mocker.patch('qutebrowser.config.configinit.msgbox.msgbox',
+                               autospec=True)
+
+    configinit.late_init(fake_save_manager)
+
+    fake_save_manager.add_saveable.assert_any_call(
+        'state-config', unittest.mock.ANY)
+    fake_save_manager.add_saveable.assert_any_call(
+        'yaml-config', unittest.mock.ANY, unittest.mock.ANY)
+    if errors:
+        assert len(msgbox_mock.call_args_list) == 1
+        _call_posargs, call_kwargs = msgbox_mock.call_args_list[0]
+        text = call_kwargs['text'].strip()
+        assert text.startswith('Errors occurred while reading config.py:')
+        assert '<b>Error text</b>: Exception' in text
+    else:
+        assert not msgbox_mock.called
+
+
+class TestQtArgs:
+
+    @pytest.fixture
+    def parser(self, mocker):
+        """Fixture to provide an argparser.
+
+        Monkey-patches .exit() of the argparser so it doesn't exit on errors.
+        """
+        parser = qutebrowser.get_argparser()
+        mocker.patch.object(parser, 'exit', side_effect=Exception)
+        return parser
+
+    @pytest.mark.parametrize('args, expected', [
+        # No Qt arguments
+        (['--debug'], [sys.argv[0]]),
+        # Qt flag
+        (['--debug', '--qt-flag', 'reverse'], [sys.argv[0], '--reverse']),
+        # Qt argument with value
+        (['--qt-arg', 'stylesheet', 'foo'],
+         [sys.argv[0], '--stylesheet', 'foo']),
+        # --qt-arg given twice
+        (['--qt-arg', 'stylesheet', 'foo', '--qt-arg', 'geometry', 'bar'],
+         [sys.argv[0], '--stylesheet', 'foo', '--geometry', 'bar']),
+        # --qt-flag given twice
+        (['--qt-flag', 'foo', '--qt-flag', 'bar'],
+         [sys.argv[0], '--foo', '--bar']),
+    ])
+    def test_qt_args(self, config_stub, args, expected, parser):
+        """Test commandline with no Qt arguments given."""
+        parsed = parser.parse_args(args)
+        assert configinit.qt_args(parsed) == expected
+
+    def test_qt_both(self, config_stub, parser):
+        """Test commandline with a Qt argument and flag."""
+        args = parser.parse_args(['--qt-arg', 'stylesheet', 'foobar',
+                                  '--qt-flag', 'reverse'])
+        qt_args = configinit.qt_args(args)
+        assert qt_args[0] == sys.argv[0]
+        assert '--reverse' in qt_args
+        assert '--stylesheet' in qt_args
+        assert 'foobar' in qt_args
+
+    def test_with_settings(self, config_stub, parser):
+        parsed = parser.parse_args(['--qt-flag', 'foo'])
+        config_stub.val.qt_args = ['bar']
+        assert configinit.qt_args(parsed) == [sys.argv[0], '--foo', '--bar']
+
+
+@pytest.mark.parametrize('arg, confval, can_import, is_new_webkit, used', [
+    # overridden by commandline arg
+    ('webkit', 'auto', False, False, usertypes.Backend.QtWebKit),
+    # overridden by config
+    (None, 'webkit', False, False, usertypes.Backend.QtWebKit),
+    # WebKit available but too old
+    (None, 'auto', True, False, usertypes.Backend.QtWebEngine),
+    # WebKit available and new
+    (None, 'auto', True, True, usertypes.Backend.QtWebKit),
+    # WebKit unavailable
+    (None, 'auto', False, False, usertypes.Backend.QtWebEngine),
+])
+def test_get_backend(monkeypatch, fake_args, config_stub,
+                     arg, confval, can_import, is_new_webkit, used):
+    real_import = __import__
+
+    def fake_import(name, *args, **kwargs):
+        if name != 'PyQt5.QtWebKit':
+            return real_import(name, *args, **kwargs)
+        if can_import:
+            return None
+        raise ImportError
+
+    fake_args.backend = arg
+    config_stub.val.backend = confval
+    monkeypatch.setattr(configinit.qtutils, 'is_new_qtwebkit',
+                        lambda: is_new_webkit)
+    monkeypatch.setattr('builtins.__import__', fake_import)
+
+    assert configinit.get_backend(fake_args) == used

tests/unit/config/test_configtypes.py

@@ -718,8 +718,10 @@ class TestBool:
     def test_to_str(self, klass, val, expected):
         assert klass().to_str(val) == expected
 
-    def test_to_doc(self, klass):
+    @pytest.mark.parametrize('value, expected', [(True, '+pass:[true]+'),
-        assert klass().to_doc(True) == '+pass:[true]+'
+                                                 (False, '+pass:[false]+')])
+    def test_to_doc(self, klass, value, expected):
+        assert klass().to_doc(value) == expected
 
 
 class TestBoolAsk:
@@ -1072,37 +1074,10 @@ class TestCommand:
         monkeypatch.setattr(configtypes, 'cmdutils', cmd_utils)
         monkeypatch.setattr('qutebrowser.commands.runners.cmdutils', cmd_utils)
 
-    @pytest.fixture(autouse=True)
-    def patch_aliases(self, config_stub):
-        """Patch the aliases setting."""
-        configtypes.Command.unvalidated = True
-        config_stub.val.aliases = {'alias': 'cmd1'}
-        configtypes.Command.unvalidated = False
-
     @pytest.fixture
     def klass(self):
         return configtypes.Command
 
-    @pytest.mark.parametrize('val', ['cmd1', 'cmd2', 'cmd1  foo bar',
-                                     'cmd2  baz fish', 'alias foo'])
-    def test_to_py_valid(self, patch_cmdutils, klass, val):
-        expected = None if not val else val
-        assert klass().to_py(val) == expected
-
-    @pytest.mark.parametrize('val', ['cmd3', 'cmd3  foo bar', ' '])
-    def test_to_py_invalid(self, patch_cmdutils, klass, val):
-        with pytest.raises(configexc.ValidationError):
-            klass().to_py(val)
-
-    def test_cmdline(self, klass, cmdline_test):
-        """Test some commandlines from the cmdline_test fixture."""
-        typ = klass()
-        if cmdline_test.valid:
-            typ.to_py(cmdline_test.cmd)
-        else:
-            with pytest.raises(configexc.ValidationError):
-                typ.to_py(cmdline_test.cmd)
-
     def test_complete(self, patch_cmdutils, klass):
         """Test completion."""
         items = klass().complete()

tests/unit/keyinput/conftest.py

@@ -31,6 +31,12 @@ BINDINGS = {'prompt': {'<Ctrl-a>': 'message-info ctrla',
             'command': {'foo': 'message-info bar',
                         '<Ctrl+X>': 'message-info ctrlx'},
             'normal': {'a': 'message-info a', 'ba': 'message-info ba'}}
+MAPPINGS = {
+    '<Ctrl+a>': 'a',
+    '<Ctrl+b>': '<Ctrl+a>',
+    'x': 'a',
+    'b': 'a',
+}
 
 
 @pytest.fixture
@@ -38,3 +44,4 @@ def keyinput_bindings(config_stub, key_config_stub):
     """Register some test bindings."""
     config_stub.val.bindings.default = {}
     config_stub.val.bindings.commands = dict(BINDINGS)
+    config_stub.val.bindings.key_mappings = dict(MAPPINGS)

tests/unit/keyinput/test_basekeyparser.py

@@ -19,7 +19,6 @@
 
 """Tests for BaseKeyParser."""
 
-import sys
 import logging
 from unittest import mock
 
@@ -92,8 +91,7 @@ class TestDebugLog:
 ])
 def test_split_count(config_stub, input_key, supports_count, expected):
     kp = basekeyparser.BaseKeyParser(0, supports_count=supports_count)
-    kp._keystring = input_key
+    assert kp._split_count(input_key) == expected
-    assert kp._split_count() == expected
 
 
 @pytest.mark.usefixtures('keyinput_bindings')
@@ -166,20 +164,14 @@ class TestSpecialKeys:
         keyparser._read_config('prompt')
 
     def test_valid_key(self, fake_keyevent_factory, keyparser):
-        if sys.platform == 'darwin':
+        modifier = Qt.MetaModifier if utils.is_mac else Qt.ControlModifier
-            modifier = Qt.MetaModifier
-        else:
-            modifier = Qt.ControlModifier
         keyparser.handle(fake_keyevent_factory(Qt.Key_A, modifier))
         keyparser.handle(fake_keyevent_factory(Qt.Key_X, modifier))
         keyparser.execute.assert_called_once_with(
             'message-info ctrla', keyparser.Type.special, None)
 
     def test_valid_key_count(self, fake_keyevent_factory, keyparser):
-        if sys.platform == 'darwin':
+        modifier = Qt.MetaModifier if utils.is_mac else Qt.ControlModifier
-            modifier = Qt.MetaModifier
-        else:
-            modifier = Qt.ControlModifier
         keyparser.handle(fake_keyevent_factory(5, text='5'))
         keyparser.handle(fake_keyevent_factory(Qt.Key_A, modifier, text='A'))
         keyparser.execute.assert_called_once_with(
@@ -200,6 +192,22 @@ class TestSpecialKeys:
         keyparser.handle(fake_keyevent_factory(Qt.Key_A, Qt.NoModifier))
         assert not keyparser.execute.called
 
+    def test_mapping(self, config_stub, fake_keyevent_factory, keyparser):
+        modifier = Qt.MetaModifier if utils.is_mac else Qt.ControlModifier
+
+        keyparser.handle(fake_keyevent_factory(Qt.Key_B, modifier))
+        keyparser.execute.assert_called_once_with(
+            'message-info ctrla', keyparser.Type.special, None)
+
+    def test_binding_and_mapping(self, config_stub, fake_keyevent_factory,
+                                 keyparser):
+        """with a conflicting binding/mapping, the binding should win."""
+        modifier = Qt.MetaModifier if utils.is_mac else Qt.ControlModifier
+
+        keyparser.handle(fake_keyevent_factory(Qt.Key_A, modifier))
+        keyparser.execute.assert_called_once_with(
+            'message-info ctrla', keyparser.Type.special, None)
+
 
 class TestKeyChain:
 
@@ -210,7 +218,7 @@ class TestKeyChain:
         keyparser._read_config('prompt')
 
     def test_valid_special_key(self, fake_keyevent_factory, keyparser):
-        if sys.platform == 'darwin':
+        if utils.is_mac:
             modifier = Qt.MetaModifier
         else:
             modifier = Qt.ControlModifier
@@ -231,7 +239,7 @@ class TestKeyChain:
         handle_text((Qt.Key_X, 'x'),
                     # Then start the real chain
                     (Qt.Key_B, 'b'), (Qt.Key_A, 'a'))
-        keyparser.execute.assert_called_once_with(
+        keyparser.execute.assert_called_with(
             'message-info ba', keyparser.Type.chain, None)
         assert keyparser._keystring == ''
 
@@ -250,6 +258,16 @@ class TestKeyChain:
         handle_text((Qt.Key_C, 'c'))
         assert keyparser._keystring == ''
 
+    def test_mapping(self, config_stub, handle_text, keyparser):
+        handle_text((Qt.Key_X, 'x'))
+        keyparser.execute.assert_called_once_with(
+            'message-info a', keyparser.Type.chain, None)
+
+    def test_binding_and_mapping(self, config_stub, handle_text, keyparser):
+        """with a conflicting binding/mapping, the binding should win."""
+        handle_text((Qt.Key_B, 'b'))
+        assert not keyparser.execute.called
+
 
 class TestCount:
 

tests/unit/keyinput/test_modeparsers.py

@@ -56,7 +56,7 @@ class TestsNormalKeyParser:
         # Then start the real chain
         keyparser.handle(fake_keyevent_factory(Qt.Key_B, text='b'))
         keyparser.handle(fake_keyevent_factory(Qt.Key_A, text='a'))
-        keyparser.execute.assert_called_once_with(
+        keyparser.execute.assert_called_with(
             'message-info ba', keyparser.Type.chain, None)
         assert keyparser._keystring == ''
 

tests/unit/misc/test_ipc.py

@@ -19,7 +19,6 @@
 
 """Tests for qutebrowser.misc.ipc."""
 
-import sys
 import os
 import getpass
 import logging
@@ -35,7 +34,7 @@ from PyQt5.QtTest import QSignalSpy
 
 import qutebrowser
 from qutebrowser.misc import ipc
-from qutebrowser.utils import objreg, standarddir
+from qutebrowser.utils import objreg, standarddir, utils
 from helpers import stubs
 
 
@@ -228,11 +227,11 @@ class TestSocketName:
         We probably would adjust the code first to make it work on that
         platform.
         """
-        if os.name == 'nt':
+        if utils.is_windows:
             pass
-        elif sys.platform == 'darwin':
+        elif utils.is_mac:
             pass
-        elif sys.platform.startswith('linux'):
+        elif utils.is_linux:
             pass
         else:
             raise Exception("Unexpected platform!")
@@ -381,7 +380,7 @@ class TestHandleConnection:
         monkeypatch.setattr(ipc_server._server, 'nextPendingConnection', m)
         ipc_server.ignored = True
         ipc_server.handle_connection()
-        assert not m.called
+        m.assert_not_called()
 
     def test_no_connection(self, ipc_server, caplog):
         ipc_server.handle_connection()
@@ -431,7 +430,7 @@ class TestHandleConnection:
 
 @pytest.fixture
 def connected_socket(qtbot, qlocalsocket, ipc_server):
-    if sys.platform == 'darwin':
+    if utils.is_mac:
         pytest.skip("Skipping connected_socket test - "
                     "https://github.com/qutebrowser/qutebrowser/issues/1045")
     ipc_server.listen()

tests/unit/misc/test_msgbox.py

@@ -18,11 +18,10 @@
 
 """Tests for qutebrowser.misc.msgbox."""
 
-import sys
-
 import pytest
 
 from qutebrowser.misc import msgbox
+from qutebrowser.utils import utils
 
 from PyQt5.QtCore import Qt
 from PyQt5.QtWidgets import QMessageBox, QWidget
@@ -40,7 +39,7 @@ def test_attributes(qtbot):
     box = msgbox.msgbox(parent=parent, title=title, text=text, icon=icon,
                         buttons=buttons)
     qtbot.add_widget(box)
-    if sys.platform != 'darwin':
+    if not utils.is_mac:
         assert box.windowTitle() == title
     assert box.icon() == icon
     assert box.standardButtons() == buttons
@@ -82,7 +81,7 @@ def test_finished_signal(qtbot):
 def test_information(qtbot):
     box = msgbox.information(parent=None, title='foo', text='bar')
     qtbot.add_widget(box)
-    if sys.platform != 'darwin':
+    if not utils.is_mac:
         assert box.windowTitle() == 'foo'
     assert box.text() == 'bar'
     assert box.icon() == QMessageBox.Information

tests/unit/misc/test_utilcmds.py

@@ -21,7 +21,6 @@
 
 import contextlib
 import logging
-import os
 import signal
 import time
 
@@ -29,6 +28,7 @@ import pytest
 
 from qutebrowser.misc import utilcmds
 from qutebrowser.commands import cmdexc
+from qutebrowser.utils import utils
 
 
 @contextlib.contextmanager
@@ -45,7 +45,7 @@ def test_debug_crash_exception():
         utilcmds.debug_crash(typ='exception')
 
 
-@pytest.mark.skipif(os.name == 'nt',
+@pytest.mark.skipif(utils.is_windows,
                     reason="current CPython/win can't recover from SIGSEGV")
 def test_debug_crash_segfault():
     """Verify that debug_crash crashes as intended."""

tests/unit/scripts/test_check_coverage.py

@@ -207,8 +207,8 @@ def test_skipped_args(covtest, args, reason):
     covtest.check_skipped(args, reason)
 
 
-def test_skipped_windows(covtest, monkeypatch):
+@pytest.mark.fake_os('windows')
-    monkeypatch.setattr(check_coverage.sys, 'platform', 'toaster')
+def test_skipped_non_linux(covtest):
     covtest.check_skipped([], "on non-Linux system.")
 
 

tests/unit/utils/test_error.py

@@ -18,12 +18,11 @@
 
 """Tests for qutebrowser.utils.error."""
 
-import sys
 import logging
 
 import pytest
 
-from qutebrowser.utils import error
+from qutebrowser.utils import error, utils
 from qutebrowser.misc import ipc
 
 from PyQt5.QtCore import QTimer
@@ -84,7 +83,7 @@ def test_err_windows(qtbot, qapp, fake_args, pre_text, post_text, expected):
         w = qapp.activeModalWidget()
         try:
             qtbot.add_widget(w)
-            if sys.platform != 'darwin':
+            if not utils.is_mac:
                 assert w.windowTitle() == 'title'
             assert w.icon() == QMessageBox.Critical
             assert w.standardButtons() == QMessageBox.Ok

tests/unit/utils/test_jinja.py

@@ -89,7 +89,7 @@ def test_resource_url():
 
     path = url.path()
 
-    if os.name == "nt":
+    if utils.is_windows:
         path = path.lstrip('/')
         path = path.replace('/', os.sep)
 

tests/unit/utils/test_qtutils.py

@@ -21,7 +21,6 @@
 
 import io
 import os
-import sys
 import os.path
 import unittest
 import unittest.mock
@@ -36,7 +35,7 @@ import pytest
 from PyQt5.QtCore import (QDataStream, QPoint, QUrl, QByteArray, QIODevice,
                           QTimer, QBuffer, QFile, QProcess, QFileDevice)
 
-from qutebrowser.utils import qtutils
+from qutebrowser.utils import qtutils, utils
 import overflow_test_cases
 
 
@@ -458,13 +457,13 @@ class TestSavefileOpen:
         with qtutils.savefile_open(str(filename)) as f:
             f.write('foo\nbar\nbaz')
         data = filename.read_binary()
-        if os.name == 'nt':
+        if utils.is_windows:
             assert data == b'foo\r\nbar\r\nbaz'
         else:
             assert data == b'foo\nbar\nbaz'
 
 
-if test_file is not None and sys.platform != 'darwin':
+if test_file is not None and not utils.is_mac:
     # If we were able to import Python's test_file module, we run some code
     # here which defines unittest TestCases to run the python tests over
     # PyQIODevice.

tests/unit/utils/test_standarddir.py

@@ -32,7 +32,7 @@ import attr
 from PyQt5.QtCore import QStandardPaths
 import pytest
 
-from qutebrowser.utils import standarddir
+from qutebrowser.utils import standarddir, utils
 
 
 # Use a different application name for tests to make sure we don't change real
@@ -78,9 +78,9 @@ def test_unset_organization_no_qapp(monkeypatch):
         pass
 
 
+@pytest.mark.fake_os('mac')
 def test_fake_mac_config(tmpdir, monkeypatch):
     """Test standardir.config on a fake Mac."""
-    monkeypatch.setattr(sys, 'platform', 'darwin')
     monkeypatch.setenv('HOME', str(tmpdir))
     expected = str(tmpdir) + '/.qute_test'  # always with /
     standarddir._init_config(args=None)
@@ -89,9 +89,9 @@ def test_fake_mac_config(tmpdir, monkeypatch):
 
 @pytest.mark.parametrize('what', ['data', 'config', 'cache'])
 @pytest.mark.not_mac
+@pytest.mark.fake_os('windows')
 def test_fake_windows(tmpdir, monkeypatch, what):
     """Make sure the config/data/cache dirs are correct on a fake Windows."""
-    monkeypatch.setattr(os, 'name', 'nt')
     monkeypatch.setattr(standarddir.QStandardPaths, 'writableLocation',
                         lambda typ: str(tmpdir / APPNAME))
 
@@ -173,9 +173,9 @@ class TestStandardDir:
         standarddir._init_dirs()
         assert standarddir.runtime() == str(tmpdir / 'temp' / APPNAME)
 
+    @pytest.mark.fake_os('windows')
     def test_runtimedir_empty_tempdir(self, monkeypatch, tmpdir):
         """With an empty tempdir on non-Linux, we should raise."""
-        monkeypatch.setattr(standarddir.sys, 'platform', 'nt')
         monkeypatch.setattr(standarddir.QStandardPaths, 'writableLocation',
                             lambda typ: '')
         with pytest.raises(standarddir.EmptyValueError):
@@ -294,7 +294,7 @@ class TestCreatingDir:
 
         assert basedir.exists()
 
-        if os.name == 'posix':
+        if utils.is_posix:
             assert basedir.stat().mode & 0o777 == 0o700
 
     @pytest.mark.parametrize('typ', DIR_TYPES)
@@ -324,9 +324,9 @@ class TestSystemData:
 
     """Test system data path."""
 
+    @pytest.mark.linux
     def test_system_datadir_exist_linux(self, monkeypatch):
         """Test that /usr/share/qute_test is used if path exists."""
-        monkeypatch.setattr('sys.platform', "linux")
         monkeypatch.setattr(os.path, 'exists', lambda path: True)
         standarddir._init_dirs()
         assert standarddir.data(system=True) == "/usr/share/qute_test"
@@ -493,18 +493,18 @@ def test_init(mocker, tmpdir, args_kind):
 
     assert standarddir._locations != {}
     if args_kind == 'normal':
-        if sys.platform == 'darwin':
+        if utils.is_mac:
-            assert not m_windows.called
+            m_windows.assert_not_called()
             assert m_mac.called
-        elif os.name == 'nt':
+        elif utils.is_windows:
             assert m_windows.called
-            assert not m_mac.called
+            m_mac.assert_not_called()
         else:
-            assert not m_windows.called
+            m_windows.assert_not_called()
-            assert not m_mac.called
+            m_mac.assert_not_called()
     else:
-        assert not m_windows.called
+        m_windows.assert_not_called()
-        assert not m_mac.called
+        m_mac.assert_not_called()
 
 
 @pytest.mark.linux

tests/unit/utils/test_utils.py

@@ -355,7 +355,7 @@ class TestKeyEventToString:
     def test_key_and_modifier(self, fake_keyevent_factory):
         """Test with key and modifier pressed."""
         evt = fake_keyevent_factory(key=Qt.Key_A, modifiers=Qt.ControlModifier)
-        expected = 'meta+a' if sys.platform == 'darwin' else 'ctrl+a'
+        expected = 'meta+a' if utils.is_mac else 'ctrl+a'
         assert utils.keyevent_to_string(evt) == expected
 
     def test_key_and_modifiers(self, fake_keyevent_factory):
@@ -365,9 +365,9 @@ class TestKeyEventToString:
                                      Qt.MetaModifier | Qt.ShiftModifier))
         assert utils.keyevent_to_string(evt) == 'ctrl+alt+meta+shift+a'
 
-    def test_mac(self, monkeypatch, fake_keyevent_factory):
+    @pytest.mark.fake_os('mac')
+    def test_mac(self, fake_keyevent_factory):
         """Test with a simulated mac."""
-        monkeypatch.setattr(sys, 'platform', 'darwin')
         evt = fake_keyevent_factory(key=Qt.Key_A, modifiers=Qt.ControlModifier)
         assert utils.keyevent_to_string(evt) == 'meta+a'
 

tests/unit/utils/test_version.py

@@ -36,7 +36,7 @@ import attr
 import pytest
 
 import qutebrowser
-from qutebrowser.utils import version, usertypes
+from qutebrowser.utils import version, usertypes, utils
 from qutebrowser.browser import pdfjs
 
 
@@ -333,7 +333,7 @@ class TestGitStrSubprocess:
                 'GIT_COMMITTER_EMAIL': 'mail@qutebrowser.org',
                 'GIT_COMMITTER_DATE': 'Thu  1 Jan 01:00:00 CET 1970',
             })
-            if os.name == 'nt':
+            if utils.is_windows:
                 # If we don't call this with shell=True it might fail under
                 # some environments on Windows...
                 # http://bugs.python.org/issue24493
@@ -662,12 +662,12 @@ class TestOsInfo:
 
     """Tests for _os_info."""
 
+    @pytest.mark.fake_os('linux')
     def test_linux_fake(self, monkeypatch):
         """Test with a fake Linux.
 
         No args because osver is set to '' if the OS is linux.
         """
-        monkeypatch.setattr(version.sys, 'platform', 'linux')
         monkeypatch.setattr(version, '_release_info',
                             lambda: [('releaseinfo', 'Hello World')])
         ret = version._os_info()
@@ -675,15 +675,16 @@ class TestOsInfo:
                     '--- releaseinfo ---', 'Hello World']
         assert ret == expected
 
+    @pytest.mark.fake_os('windows')
     def test_windows_fake(self, monkeypatch):
         """Test with a fake Windows."""
-        monkeypatch.setattr(version.sys, 'platform', 'win32')
         monkeypatch.setattr(version.platform, 'win32_ver',
                             lambda: ('eggs', 'bacon', 'ham', 'spam'))
         ret = version._os_info()
         expected = ['OS Version: eggs, bacon, ham, spam']
         assert ret == expected
 
+    @pytest.mark.fake_os('mac')
     @pytest.mark.parametrize('mac_ver, mac_ver_str', [
         (('x', ('', '', ''), 'y'), 'x, y'),
         (('', ('', '', ''), ''), ''),
@@ -696,15 +697,14 @@ class TestOsInfo:
             mac_ver: The tuple to set platform.mac_ver() to.
             mac_ver_str: The expected Mac version string in version._os_info().
         """
-        monkeypatch.setattr(version.sys, 'platform', 'darwin')
         monkeypatch.setattr(version.platform, 'mac_ver', lambda: mac_ver)
         ret = version._os_info()
         expected = ['OS Version: {}'.format(mac_ver_str)]
         assert ret == expected
 
-    def test_unknown_fake(self, monkeypatch):
+    @pytest.mark.fake_os('unknown')
-        """Test with a fake unknown sys.platform."""
+    def test_unknown_fake(self):
-        monkeypatch.setattr(version.sys, 'platform', 'toaster')
+        """Test with a fake unknown platform."""
         ret = version._os_info()
         expected = ['OS Version: ?']
         assert ret == expected