Merge branch 'master' of https://github.com/qutebrowser/qutebrowser into jay/insert-no-leave

.flake8

@@ -46,12 +46,11 @@ ignore =
 min-version = 3.4.0
 max-complexity = 12
 per-file-ignores =
-    /qutebrowser/api/hook.py : N801
+    qutebrowser/api/hook.py : N801
-    /tests/**/*.py : D100,D101,D401
+    tests/* : D100,D101
-    /tests/unit/browser/test_history.py : N806
+    tests/unit/browser/test_history.py : D100,D101,N806
-    /tests/helpers/fixtures.py : N806
+    tests/helpers/fixtures.py : D100,D101,N806
-    /tests/unit/browser/webkit/http/test_content_disposition.py : D400
+    tests/unit/browser/webkit/http/test_content_disposition.py : D100,D101,D400
-    /scripts/dev/ci/appveyor_install.py : FI53
 copyright-check = True
 copyright-regexp = # Copyright [\d-]+ .*
 copyright-min-file-size = 110

.travis.yml

@@ -5,51 +5,84 @@ python: 3.6
 os: linux
 
 matrix:
+  fast_finish: true
   include:
+    ### Archlinux QtWebKit
     - env: DOCKER=archlinux
       services: docker
+
+    ### Archlinux QtWebEngine
     - env: DOCKER=archlinux-webengine QUTE_BDD_WEBENGINE=true
       services: docker
-    - env: TESTENV=py36-pyqt571
+
+    ### PyQt 5.7.1 (Python 3.5)
     - python: 3.5
       env: TESTENV=py35-pyqt571
+    ### PyQt 5.7.1
+    - env: TESTENV=py36-pyqt571
+
+    ### PyQt 5.9
     - env: TESTENV=py36-pyqt59
+
+    ### PyQt 5.10
     - env: TESTENV=py36-pyqt510
       addons:
         apt:
           packages:
           - xfonts-base
+
+    ### PyQt 5.11 (with coverage)
     - env: TESTENV=py36-pyqt511-cov
+    ### PyQt 5.11 (Python 3.7)
     - python: 3.7
       env: TESTENV=py37-pyqt511
+
+    ### PyQt 5.12
+    - env: TESTENV=py36-pyqt512
+      addons:
+        apt:
+          packages:
+          - libxkbcommon-x11-0
+
+    ### macOS sierra
     - os: osx
-      env: TESTENV=py37 OSX=sierra
+      env: TESTENV=py37-pyqt511 OSX=sierra
       osx_image: xcode9.2
       language: generic
+    ### macOS yosemite
     # https://github.com/qutebrowser/qutebrowser/issues/2013
     # - os: osx
     #   env: TESTENV=py35 OSX=yosemite
     #   osx_image: xcode6.4
+
+    ### pylint/flake8/mypy
     - env: TESTENV=pylint
     - env: TESTENV=flake8
     - env: TESTENV=mypy
+
+    ### docs
     - env: TESTENV=docs
       addons:
          apt:
            packages:
            - asciidoc
+
+    ### vulture/misc/pyroma/check-manifest
     - env: TESTENV=vulture
     - env: TESTENV=misc
     - env: TESTENV=pyroma
     - env: TESTENV=check-manifest
+
+    ### eslint
     - env: TESTENV=eslint
       language: node_js
       python: null
       node_js: "lts/*"
+
+    ### shellcheck
     - language: generic
       env: TESTENV=shellcheck
       services: docker
-  fast_finish: true
 
 cache:
   directories:

doc/changelog.asciidoc

@@ -35,6 +35,9 @@ Added
   are used for hints, and also allows adding custom hint groups.
 - New `:yank markdown` feature which yanks the current URL and title in
   markdown format.
+- Basic support for client certificates with Qt 5.12. Selecting the certificate
+  to show when there are multiple matching certificates isn't implemented yet.
+- Support for DNS prefetching (`content.dns_prefetch`) with QtWebEngine on 5.12.
 
 Changed
 ~~~~~~~
@@ -59,6 +62,9 @@ Changed
   `org.qutebrowser.qutebrowser.appdata.xml`.
 - The `qute-pass` userscript now understands domains in gpg filenames
   in addition to directory names.
+- macOS: The IPC socket path used to communicate with existing instances
+  changed due to changes in Qt 5.12. Please make sure to quit qutebrowser
+  before upgrading.
 
 Fixed
 ~~~~~
@@ -79,6 +85,11 @@ Fixed
 - When `scrolling.bar = True` was set in versions before v1.5.0, this now
   correctly gets migrated to `always` instead of `when-searching`.
 - Completion highlighting now works again on Qt 5.11.3 and 5.12.1.
+- The outdated header `X-Do-Not-Track` is no longer sent.
+- A javascript error on page load when using Qt 5.12 was fixed.
+- `window.print()` works with Qt 5.12 now.
+- PAC proxies were never correctly supported with QtWebEngine, but are now
+  explicitly disallowed.
 
 v1.5.2
 ------
@@ -378,11 +389,11 @@ v1.3.3
 Security
 ~~~~~~~~
 
-- An XSS vulnerability on the `qute://history` page allowed websites to inject
+- CVE-2018-1000559: An XSS vulnerability on the `qute://history` page allowed
-  HTML into the page via a crafted title tag. This could allow them to steal
+  websites to inject HTML into the page via a crafted title tag. This could
-  your browsing history. If you're currently unable to upgrade, avoid using
+  allow them to steal your browsing history. If you're currently unable to
-  `:history`. A CVE request for this issue is pending, see
+  upgrade, avoid using `:history`. See the related GitHub issue for details:
-  https://github.com/qutebrowser/qutebrowser/issues/4011[#4011] for updates.
+  https://github.com/qutebrowser/qutebrowser/issues/4011.
 
 Fixed
 ~~~~~

doc/contributing.asciidoc

@@ -710,6 +710,7 @@ qutebrowser release
 
 * Update changelog (remove *(unreleased)*).
 * Adjust `__version_info__` in `qutebrowser/__init__.py`.
+* Consider updating the completions for `content.headers.user_agent` in `configdata.yml`.
 * Commit.
 
 * Create annotated git tag (`git tag -s "v1.$x.$y" -m "Release v1.$x.$y"`).

doc/help/settings.asciidoc

@@ -1647,7 +1647,7 @@ Type: <<types,Bool>>
 
 Default: +pass:[true]+
 
-This setting is only available with the QtWebKit backend.
+On QtWebEngine, this setting requires Qt 5.12 or newer.
 
 [[content.frame_flattening]]
 === content.frame_flattening

misc/requirements/requirements-flake8.txt

@@ -1,27 +1,25 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 attrs==18.2.0
-flake8==3.6.0
+entrypoints==0.3
+flake8==3.7.5
 flake8-bugbear==18.8.0
 flake8-builtins==1.4.1
-flake8-comprehensions==1.4.1
+flake8-comprehensions==2.0.0
 flake8-copyright==0.2.2
 flake8-debugger==3.1.0
 flake8-deprecated==1.3
 flake8-docstrings==1.3.0
 flake8-future-import==0.4.5
 flake8-mock==0.3
-flake8-per-file-ignores==0.7
 flake8-polyfill==1.0.2
 flake8-string-format==0.2.3
-flake8-tidy-imports==1.1.0
+flake8-tidy-imports==2.0.0
 flake8-tuple==0.2.13
 mccabe==0.6.1
-pathmatch==0.2.1
+pep8-naming==0.8.2
-pep8-naming==0.7.0
+pycodestyle==2.5.0
-pycodestyle==2.4.0
 pydocstyle==3.0.0
-pyflakes==2.0.0
+pyflakes==2.1.0
 six==1.12.0
 snowballstemmer==1.2.1
-typing==3.6.6

misc/requirements/requirements-flake8.txt-raw

@@ -8,7 +8,6 @@ flake8-deprecated
 flake8-docstrings
 flake8-future-import
 flake8-mock
-flake8-per-file-ignores
 flake8-string-format
 flake8-tidy-imports
 flake8-tuple

misc/requirements/requirements-mypy.txt

@@ -1,8 +1,8 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-mypy==0.650
+mypy==0.670
 mypy-extensions==0.4.1
-PyQt5==5.11.3
+# PyQt5==5.11.3
-PyQt5-sip==4.19.13
+# PyQt5-sip==4.19.14
 -e git+https://github.com/qutebrowser/PyQt5-stubs.git@wip#egg=PyQt5_stubs
-typed-ast==1.1.1
+typed-ast==1.3.1

misc/requirements/requirements-mypy.txt-raw

@@ -3,3 +3,4 @@ mypy
 
 # remove @commit-id for scm installs
 #@ replace: @.*# @wip#
+#@ ignore: PyQt5, PyQt5-sip

misc/requirements/requirements-optional.txt

@@ -2,6 +2,5 @@
 
 colorama==0.4.1
 cssutils==1.0.2
-hunter==2.1.0
+hunter==2.2.1
 Pympler==0.6
-six==1.12.0

misc/requirements/requirements-pip.txt

@@ -1,8 +1,8 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 appdirs==1.4.3
-packaging==18.0
+packaging==19.0
-pyparsing==2.3.0
+pyparsing==2.3.1
-setuptools==40.6.3
+setuptools==40.8.0
 six==1.12.0
 wheel==0.32.3

misc/requirements/requirements-pylint.txt

@@ -5,8 +5,8 @@ astroid==2.1.0
 certifi==2018.11.29
 cffi==1.11.5
 chardet==3.0.4
-cryptography==2.4.2
+cryptography==2.5
-github3.py==1.2.0
+github3.py==1.3.0
 idna==2.8
 isort==4.3.4
 jwcrypto==0.6.0
@@ -14,10 +14,10 @@ lazy-object-proxy==1.3.1
 mccabe==0.6.1
 pycparser==2.19
 pylint==2.2.2
-python-dateutil==2.7.5
+python-dateutil==2.8.0
 ./scripts/dev/pylint_checkers
 requests==2.21.0
 six==1.12.0
 uritemplate==3.0.0
 urllib3==1.24.1
-wrapt==1.10.11
+wrapt==1.11.1

misc/requirements/requirements-pyqt.txt

@@ -1,4 +1,5 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-PyQt5==5.11.3
+PyQt5==5.12
-PyQt5-sip==4.19.13
+PyQt5-sip==4.19.14
+PyQtWebEngine==5.12

misc/requirements/requirements-pyqt.txt-raw

@@ -1 +1,2 @@
 PyQt5
+PyQtWebEngine

misc/requirements/requirements-sphinx.txt

@@ -9,13 +9,13 @@ idna==2.8
 imagesize==1.1.0
 Jinja2==2.10
 MarkupSafe==1.1.0
-packaging==18.0
+packaging==19.0
 Pygments==2.3.1
-pyparsing==2.3.0
+pyparsing==2.3.1
-pytz==2018.7
+pytz==2018.9
 requests==2.21.0
 six==1.12.0
 snowballstemmer==1.2.1
-Sphinx==1.8.3
+Sphinx==1.8.4
 sphinxcontrib-websupport==1.1.0
 urllib3==1.24.1

misc/requirements/requirements-tests.txt

@@ -1,41 +1,42 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-atomicwrites==1.2.1
+atomicwrites==1.3.0
 attrs==18.2.0
 backports.functools-lru-cache==1.5
-beautifulsoup4==4.7.0
+beautifulsoup4==4.7.1
-cheroot==6.5.3
+cheroot==6.5.4
 Click==7.0
 # colorama==0.4.1
 coverage==4.5.2
 EasyProcess==0.2.5
 Flask==1.0.2
 glob2==0.6
-hunter==2.1.0
+hunter==2.2.1
-hypothesis==3.85.2
+hypothesis==4.5.6
 itsdangerous==1.1.0
 # Jinja2==2.10
 Mako==1.0.7
 # MarkupSafe==1.1.0
 more-itertools==5.0.0
-parse==1.9.0
+parse==1.11.1
 parse-type==0.4.2
-pluggy==0.8.0
+pluggy==0.8.1
 py==1.7.0
 py-cpuinfo==4.0.0
-pytest==4.0.2
+pytest==4.2.0
 pytest-bdd==3.0.1
-pytest-benchmark==3.1.1
+pytest-benchmark==3.2.2
-pytest-cov==2.6.0
+pytest-cov==2.6.1
 pytest-faulthandler==1.5.0
 pytest-instafail==0.4.0
-pytest-mock==1.10.0
+pytest-mock==1.10.1
 pytest-qt==3.2.2
 pytest-repeat==0.7.0
-pytest-rerunfailures==5.0
+pytest-rerunfailures==6.0
 pytest-travis-fold==1.3.0
-pytest-xvfb==1.1.0
+pytest-xvfb==1.2.0
 PyVirtualDisplay==0.2.1
 six==1.12.0
+soupsieve==1.7.3
 vulture==1.0
 Werkzeug==0.14.1

misc/requirements/requirements-tox.txt

@@ -1,9 +1,9 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 filelock==3.0.10
-pluggy==0.8.0
+pluggy==0.8.1
 py==1.7.0
 six==1.12.0
 toml==0.10.0
-tox==3.6.1
+tox==3.7.0
-virtualenv==16.1.0
+virtualenv==16.4.0

misc/userscripts/format_json

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -euo pipefail
 #
 # Behavior:

pytest.ini

@@ -65,10 +65,9 @@ qt_log_ignore =
     ^Icon theme ".*" not found
     ^Error receiving trust for a CA certificate
     ^QBackingStore::endPaint\(\) called with active painter on backingstore paint device
+    ^QPaintDevice: Cannot destroy paint device that is being painted
 xfail_strict = true
 filterwarnings =
     error
-    # This happens in many qutebrowser dependencies...
+    # WORKAROUND for https://github.com/pytest-dev/pytest-bdd/pull/288
-    ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working:DeprecationWarning
+    ignore:the `pytest\.config` global is deprecated\.  Please use `request.config` or `pytest_configure` \(if you're a pytest plugin\) instead\.
-    # WORKAROUND for https://github.com/ionelmc/pytest-benchmark/issues/124
-    ignore:Node\.warn\(code, message\) form has been deprecated, use Node\.warn\(warning_instance\) instead:pytest.PytestDeprecationWarning

qutebrowser/app.py

@@ -83,7 +83,7 @@ from qutebrowser.misc import utilcmds
 # pylint: enable=unused-import
 
 
-qApp = None
+q_app = None
 
 
 def run(args):
@@ -101,25 +101,25 @@ def run(args):
     log.init.debug("Initializing config...")
     configinit.early_init(args)
 
-    global qApp
+    global q_app
-    qApp = Application(args)
+    q_app = Application(args)
-    qApp.setOrganizationName("qutebrowser")
+    q_app.setOrganizationName("qutebrowser")
-    qApp.setApplicationName("qutebrowser")
+    q_app.setApplicationName("qutebrowser")
-    qApp.setDesktopFileName("qutebrowser")
+    q_app.setDesktopFileName("qutebrowser")
-    qApp.setApplicationVersion(qutebrowser.__version__)
+    q_app.setApplicationVersion(qutebrowser.__version__)
-    qApp.lastWindowClosed.connect(quitter.on_last_window_closed)
+    q_app.lastWindowClosed.connect(quitter.on_last_window_closed)
 
     if args.version:
         print(version.version())
         sys.exit(usertypes.Exit.ok)
 
     crash_handler = crashsignal.CrashHandler(
-        app=qApp, quitter=quitter, args=args, parent=qApp)
+        app=q_app, quitter=quitter, args=args, parent=q_app)
     crash_handler.activate()
     objreg.register('crash-handler', crash_handler)
 
-    signal_handler = crashsignal.SignalHandler(app=qApp, quitter=quitter,
+    signal_handler = crashsignal.SignalHandler(app=q_app, quitter=quitter,
-                                               parent=qApp)
+                                               parent=q_app)
     signal_handler.activate()
     objreg.register('signal-handler', signal_handler)
 
@@ -151,7 +151,7 @@ def qt_mainloop():
     WARNING: misc/crashdialog.py checks the stacktrace for this function
     name, so if this is changed, it should be changed there as well!
     """
-    return qApp.exec_()
+    return q_app.exec_()
 
 
 def init(args, crash_handler):
@@ -162,7 +162,7 @@ def init(args, crash_handler):
         crash_handler: The CrashHandler instance.
     """
     log.init.debug("Starting init...")
-    qApp.setQuitOnLastWindowClosed(False)
+    q_app.setQuitOnLastWindowClosed(False)
     _init_icon()
 
     loader.init()
@@ -175,12 +175,12 @@ def init(args, crash_handler):
         sys.exit(usertypes.Exit.err_init)
 
     log.init.debug("Initializing eventfilter...")
-    event_filter = EventFilter(qApp)
+    event_filter = EventFilter(q_app)
-    qApp.installEventFilter(event_filter)
+    q_app.installEventFilter(event_filter)
     objreg.register('event-filter', event_filter)
 
     log.init.debug("Connecting signals...")
-    qApp.focusChanged.connect(on_focus_changed)
+    q_app.focusChanged.connect(on_focus_changed)
 
     _process_args(args)
 
@@ -207,7 +207,7 @@ def _init_icon():
     if icon.isNull():
         log.init.warning("Failed to load icon")
     else:
-        qApp.setWindowIcon(icon)
+        q_app.setWindowIcon(icon)
 
 
 def _process_args(args):
@@ -220,7 +220,7 @@ def _process_args(args):
         window = mainwindow.MainWindow(private=None)
         if not args.nowindow:
             window.show()
-        qApp.setActiveWindow(window)
+        q_app.setActiveWindow(window)
 
     process_pos_args(args.command)
     _open_startpage()
@@ -425,7 +425,7 @@ def _init_modules(args, crash_handler):
         crash_handler: The CrashHandler instance.
     """
     log.init.debug("Initializing save manager...")
-    save_manager = savemanager.SaveManager(qApp)
+    save_manager = savemanager.SaveManager(q_app)
     objreg.register('save-manager', save_manager)
     configinit.late_init(save_manager)
 
@@ -450,7 +450,7 @@ def _init_modules(args, crash_handler):
         sql.init(os.path.join(standarddir.data(), 'history.sqlite'))
 
         log.init.debug("Initializing web history...")
-        history.init(qApp)
+        history.init(q_app)
     except sql.SqlEnvironmentError as e:
         error.handle_fatal_exc(e, args, 'Error initializing SQL',
                                pre_text='Error initializing SQL')
@@ -464,31 +464,31 @@ def _init_modules(args, crash_handler):
         crash_handler.handle_segfault()
 
     log.init.debug("Initializing sessions...")
-    sessions.init(qApp)
+    sessions.init(q_app)
 
     log.init.debug("Initializing websettings...")
     websettings.init(args)
 
     log.init.debug("Initializing quickmarks...")
-    quickmark_manager = urlmarks.QuickmarkManager(qApp)
+    quickmark_manager = urlmarks.QuickmarkManager(q_app)
     objreg.register('quickmark-manager', quickmark_manager)
 
     log.init.debug("Initializing bookmarks...")
-    bookmark_manager = urlmarks.BookmarkManager(qApp)
+    bookmark_manager = urlmarks.BookmarkManager(q_app)
     objreg.register('bookmark-manager', bookmark_manager)
 
     log.init.debug("Initializing cookies...")
-    cookie_jar = cookies.CookieJar(qApp)
+    cookie_jar = cookies.CookieJar(q_app)
-    ram_cookie_jar = cookies.RAMCookieJar(qApp)
+    ram_cookie_jar = cookies.RAMCookieJar(q_app)
     objreg.register('cookie-jar', cookie_jar)
     objreg.register('ram-cookie-jar', ram_cookie_jar)
 
     log.init.debug("Initializing cache...")
-    diskcache = cache.DiskCache(standarddir.cache(), parent=qApp)
+    diskcache = cache.DiskCache(standarddir.cache(), parent=q_app)
     objreg.register('cache', diskcache)
 
     log.init.debug("Initializing downloads...")
-    download_manager = qtnetworkdownloads.DownloadManager(parent=qApp)
+    download_manager = qtnetworkdownloads.DownloadManager(parent=q_app)
     objreg.register('qtnetwork-download-manager', download_manager)
 
     log.init.debug("Initializing Greasemonkey...")
@@ -735,7 +735,7 @@ class Quitter:
     def _shutdown(self, status, restart):  # noqa
         """Second stage of shutdown."""
         log.destroy.debug("Stage 2 of shutting down...")
-        if qApp is None:
+        if q_app is None:
             # No QApplication exists yet, so quit hard.
             sys.exit(status)
         # Remove eventfilter
@@ -743,7 +743,7 @@ class Quitter:
             log.destroy.debug("Removing eventfilter...")
             event_filter = objreg.get('event-filter', None)
             if event_filter is not None:
-                qApp.removeEventFilter(event_filter)
+                q_app.removeEventFilter(event_filter)
         except AttributeError:
             pass
         # Close all windows
@@ -792,7 +792,7 @@ class Quitter:
             session_manager.delete_autosave()
         # We use a singleshot timer to exit here to minimize the likelihood of
         # segfaults.
-        QTimer.singleShot(0, functools.partial(qApp.exit, status))
+        QTimer.singleShot(0, functools.partial(q_app.exit, status))
 
 
 class Application(QApplication):
@@ -893,7 +893,7 @@ class EventFilter(QObject):
         Return:
             True if the event should be filtered, False if it's passed through.
         """
-        if qApp.activeWindow() not in objreg.window_registry.values():
+        if q_app.activeWindow() not in objreg.window_registry.values():
             # Some other window (print dialog, etc.) is focused so we pass the
             # event through.
             return False

qutebrowser/browser/network/pac.py

@@ -208,6 +208,8 @@ class PACResolver:
         Return:
             A list of QNetworkProxy objects in order of preference.
         """
+        qtutils.ensure_valid(query.url())
+
         if from_file:
             string_flags = QUrl.PrettyDecoded
         else:

qutebrowser/browser/network/proxy.py

@@ -20,10 +20,12 @@
 """Handling of proxies."""
 
 
+from PyQt5.QtCore import QUrl
 from PyQt5.QtNetwork import QNetworkProxy, QNetworkProxyFactory
 
 from qutebrowser.config import config, configtypes
-from qutebrowser.utils import objreg
+from qutebrowser.utils import objreg, message, usertypes, urlutils
+from qutebrowser.misc import objects
 from qutebrowser.browser.network import pac
 
 
@@ -33,6 +35,18 @@ def init():
     objreg.register('proxy-factory', proxy_factory)
     QNetworkProxyFactory.setApplicationProxyFactory(proxy_factory)
 
+    config.instance.changed.connect(_warn_for_pac)
+    _warn_for_pac()
+
+
+@config.change_filter('content.proxy', function=True)
+def _warn_for_pac():
+    """Show a warning if PAC is used with QtWebEngine."""
+    proxy = config.val.content.proxy
+    if (isinstance(proxy, pac.PACFetcher) and
+            objects.backend == usertypes.Backend.QtWebEngine):
+        message.error("PAC support isn't implemented for QtWebEngine yet!")
+
 
 def shutdown():
     QNetworkProxyFactory.setApplicationProxyFactory(None)
@@ -70,6 +84,10 @@ class ProxyFactory(QNetworkProxyFactory):
             # ref. http://doc.qt.io/qt-5/qnetworkproxyfactory.html#systemProxyForQuery
             proxies = QNetworkProxyFactory.systemProxyForQuery(query)
         elif isinstance(proxy, pac.PACFetcher):
+            if objects.backend == usertypes.Backend.QtWebEngine:
+                # Looks like query.url() is always invalid on QtWebEngine...
+                proxies = [urlutils.proxy_from_url(QUrl('direct://'))]
+            else:
                 proxies = proxy.resolve(query)
         else:
             proxies = [proxy]

qutebrowser/browser/qutescheme.py

@@ -340,18 +340,10 @@ def qute_gpl(_url):
 
 def _asciidoc_fallback_path(html_path):
     """Fall back to plaintext asciidoc if the HTML is unavailable."""
-    asciidoc_path = html_path.replace('.html', '.asciidoc')
+    path = html_path.replace('.html', '.asciidoc')
-    asciidoc_paths = [asciidoc_path]
-    if asciidoc_path.startswith('html/doc/'):
-        asciidoc_paths += [asciidoc_path.replace('html/doc/', '../doc/help/'),
-                           asciidoc_path.replace('html/doc/', '../doc/')]
-
-    for path in asciidoc_paths:
     try:
         return utils.read_file(path)
     except OSError:
-            pass
-
         return None
 
 

qutebrowser/browser/shared.py

@@ -42,7 +42,6 @@ def custom_headers(url):
     if dnt_config is not None:
         dnt = b'1' if dnt_config else b'0'
         headers[b'DNT'] = dnt
-        headers[b'X-Do-Not-Track'] = dnt
 
     conf_headers = config.instance.get('content.headers.custom', url=url)
     for header, value in conf_headers.items():

qutebrowser/browser/webengine/webenginedownloads.py

@@ -180,7 +180,21 @@ def _get_suggested_filename(path):
     See https://bugreports.qt.io/browse/QTBUG-56978
     """
     filename = os.path.basename(path)
-    filename = re.sub(r'\([0-9]+\)(?=\.|$)', '', filename)
+
+    suffix_re = re.compile(r"""
+      \ ?  # Optional space between filename and suffix
+      (
+        # Numerical suffix
+        \([0-9]+\)
+      |
+        # ISO-8601 suffix
+        # https://cs.chromium.org/chromium/src/base/time/time_to_iso8601.cc
+        \ -\ \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z
+      )
+      (?=\.|$)  # Begin of extension, or filename without extension
+    """, re.VERBOSE)
+
+    filename = suffix_re.sub('', filename)
     if not qtutils.version_check('5.9', compiled=False):
         # https://bugreports.qt.io/browse/QTBUG-58155
         filename = urllib.parse.unquote(filename)

qutebrowser/browser/webengine/webenginequtescheme.py

@@ -62,18 +62,33 @@ class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
         """
         try:
             initiator = job.initiator()
+            request_url = job.requestUrl()
         except AttributeError:
             # Added in Qt 5.11
             return True
 
-        if initiator == QUrl('null') and not qtutils.version_check('5.12'):
+        # https://codereview.qt-project.org/#/c/234849/
+        is_opaque = initiator == QUrl('null')
+        target = request_url.scheme(), request_url.host()
+
+        if is_opaque and not qtutils.version_check('5.12'):
             # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-70421
+            # When we don't register the qute:// scheme, all requests are
+            # flagged as opaque.
+            return True
+
+        if (target == ('qute', 'testdata') and
+                is_opaque and
+                qtutils.version_check('5.12')):
+            # Allow requests to qute://testdata, as this is needed in Qt 5.12
+            # for all tests to work properly. No qute://testdata handler is
+            # installed outside of tests.
             return True
 
         if initiator.isValid() and initiator.scheme() != 'qute':
             log.misc.warning("Blocking malicious request from {} to {}".format(
                 initiator.toDisplayString(),
-                job.requestUrl().toDisplayString()))
+                request_url.toDisplayString()))
             job.fail(QWebEngineUrlRequestJob.RequestDenied)
             return False
 

qutebrowser/browser/webengine/webenginesettings.py

@@ -25,6 +25,7 @@ Module attributes:
 """
 
 import os
+import operator
 
 from PyQt5.QtGui import QFont
 from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile,
@@ -163,9 +164,14 @@ class WebEngineSettings(websettings.AbstractSettings):
             # Qt 5.8
             'content.print_element_backgrounds':
                 ('PrintElementBackgrounds', None),
+
             # Qt 5.11
             'content.autoplay':
-                ('PlaybackRequiresUserGesture', lambda val: not val),
+                ('PlaybackRequiresUserGesture', operator.not_),
+
+            # Qt 5.12
+            'content.dns_prefetch':
+                ('DnsPrefetchEnabled', None),
         }
         for name, (attribute, converter) in new_attributes.items():
             try:

qutebrowser/browser/webengine/webenginetab.py

@@ -715,8 +715,6 @@ class _WebEnginePermissions(QObject):
 
     """Handling of various permission-related signals."""
 
-    _abort_questions = pyqtSignal()
-
     def __init__(self, tab, parent=None):
         super().__init__(parent)
         self._tab = tab
@@ -736,9 +734,6 @@ class _WebEnginePermissions(QObject):
             page.registerProtocolHandlerRequested.connect(
                 self._on_register_protocol_handler_requested)
 
-        self._tab.shutting_down.connect(self._abort_questions)
-        self._tab.load_started.connect(self._abort_questions)
-
     @pyqtSlot('QWebEngineFullScreenRequest')
     def _on_fullscreen_requested(self, request):
         request.accept()
@@ -816,7 +811,7 @@ class _WebEnginePermissions(QObject):
         question = shared.feature_permission(
             url=url, option=options[feature], msg=messages[feature],
             yes_action=yes_action, no_action=no_action,
-            abort_on=[self._abort_questions])
+            abort_on=[self._tab.abort_questions])
 
         if question is not None:
             page.featurePermissionRequestCanceled.connect(
@@ -844,7 +839,7 @@ class _WebEnginePermissions(QObject):
             option='content.persistent_storage',
             msg='use {} of persistent storage'.format(size),
             yes_action=request.accept, no_action=request.reject,
-            abort_on=[self._abort_questions],
+            abort_on=[self._tab.abort_questions],
             blocking=True)
 
     def _on_register_protocol_handler_requested(self, request):
@@ -853,7 +848,7 @@ class _WebEnginePermissions(QObject):
             option='content.register_protocol_handler',
             msg='open all {} links'.format(request.scheme()),
             yes_action=request.accept, no_action=request.reject,
-            abort_on=[self._abort_questions],
+            abort_on=[self._tab.abort_questions],
             blocking=True)
 
 
@@ -927,6 +922,10 @@ class _WebEngineScripts(QObject):
             utils.read_file('javascript/webelem.js'),
             utils.read_file('javascript/caret.js'),
         )
+        if not qtutils.version_check('5.12'):
+            # WORKAROUND for Qt versions < 5.12 not exposing window.print().
+            # Qt 5.12 has a printRequested() signal so we don't need this hack
+            # anymore.
             self._inject_early_js('js',
                                   utils.read_file('javascript/print.js'),
                                   subframes=True,
@@ -1076,10 +1075,13 @@ class WebEngineTab(browsertab.AbstractTab):
     Signals:
         _load_finished_fake:
             Used in place of unreliable loadFinished
+        abort_questions: Emitted when a new load started or we're shutting
+            down.
     """
 
     # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-65223
     _load_finished_fake = pyqtSignal(bool)
+    abort_questions = pyqtSignal()
 
     def __init__(self, *, win_id, mode_manager, private, parent=None):
         super().__init__(win_id=win_id, private=private, parent=parent)
@@ -1252,15 +1254,13 @@ class WebEngineTab(browsertab.AbstractTab):
         answer = message.ask(
             title="Proxy authentication required", text=msg,
             mode=usertypes.PromptMode.user_pwd,
-            abort_on=[self.shutting_down, self.load_started], url=urlstr)
+            abort_on=[self.abort_questions], url=urlstr)
         if answer is not None:
             authenticator.setUser(answer.user)
             authenticator.setPassword(answer.password)
         else:
             try:
-                # pylint: disable=no-member, useless-suppression
                 sip.assign(authenticator, QAuthenticator())
-                # pylint: enable=no-member, useless-suppression
             except AttributeError:
                 self._show_error_page(url, "Proxy authentication required")
 
@@ -1276,15 +1276,12 @@ class WebEngineTab(browsertab.AbstractTab):
 
         if not netrc_success:
             log.network.debug("Asking for credentials")
-            abort_on = [self.shutting_down, self.load_started]
+            answer = shared.authentication_required(
-            answer = shared.authentication_required(url, authenticator,
+                url, authenticator, abort_on=[self.abort_questions])
-                                                    abort_on)
         if not netrc_success and answer is None:
             log.network.debug("Aborting auth")
             try:
-                # pylint: disable=no-member, useless-suppression
                 sip.assign(authenticator, QAuthenticator())
-                # pylint: enable=no-member, useless-suppression
             except AttributeError:
                 # WORKAROUND for
                 # https://www.riverbankcomputing.com/pipermail/pyqt/2016-December/038400.html
@@ -1389,7 +1386,7 @@ class WebEngineTab(browsertab.AbstractTab):
 
         if error.is_overridable():
             error.ignore = shared.ignore_certificate_errors(
-                url, [error], abort_on=[self.shutting_down, self.load_started])
+                url, [error], abort_on=[self.abort_questions])
         else:
             log.webview.error("Non-overridable certificate error: "
                               "{}".format(error))
@@ -1418,15 +1415,20 @@ class WebEngineTab(browsertab.AbstractTab):
         if not qtutils.version_check('5.11.1', compiled=False):
             self.settings.update_for_url(url)
 
+    @pyqtSlot()
+    def _on_print_requested(self):
+        """Slot for window.print() in JS."""
+        try:
+            self.printing.show_dialog()
+        except browsertab.WebTabError as e:
+            message.error(str(e))
+
     @pyqtSlot(usertypes.NavigationRequest)
     def _on_navigation_request(self, navigation):
         super()._on_navigation_request(navigation)
 
         if navigation.url == QUrl('qute://print'):
-            try:
+            self._on_print_requested()
-                self.printing.show_dialog()
-            except browsertab.WebTabError as e:
-                message.error(str(e))
             navigation.accepted = False
 
         if not navigation.accepted or not navigation.is_main_frame:
@@ -1458,6 +1460,37 @@ class WebEngineTab(browsertab.AbstractTab):
         if reload_needed:
             self._reload_url = navigation.url
 
+    def _on_select_client_certificate(self, selection):
+        """Handle client certificates.
+
+        Currently, we simply pick the first available certificate and show an
+        additional note if there are multiple matches.
+        """
+        certificate = selection.certificates()[0]
+        text = ('<b>Subject:</b> {subj}<br/>'
+                '<b>Issuer:</b> {issuer}<br/>'
+                '<b>Serial:</b> {serial}'.format(
+                    subj=html_utils.escape(certificate.subjectDisplayName()),
+                    issuer=html_utils.escape(certificate.issuerDisplayName()),
+                    serial=bytes(certificate.serialNumber()).decode('ascii')))
+        if len(selection.certificates()) > 1:
+            text += ('<br/><br/><b>Note:</b> Multiple matching certificates '
+                     'were found, but certificate selection is not '
+                     'implemented yet!')
+        urlstr = selection.host().host()
+
+        present = message.ask(
+            title='Present client certificate to {}?'.format(urlstr),
+            text=text,
+            mode=usertypes.PromptMode.yesno,
+            abort_on=[self.abort_questions],
+            url=urlstr)
+
+        if present:
+            selection.select(certificate)
+        else:
+            selection.selectNone()
+
     def _connect_signals(self):
         view = self._widget
         page = view.page()
@@ -1473,6 +1506,11 @@ class WebEngineTab(browsertab.AbstractTab):
         page.contentsSizeChanged.connect(self.contents_size_changed)
         page.navigation_request.connect(self._on_navigation_request)
 
+        if qtutils.version_check('5.12'):
+            page.printRequested.connect(self._on_print_requested)
+            page.selectClientCertificate.connect(
+                self._on_select_client_certificate)
+
         view.titleChanged.connect(self.title_changed)
         view.urlChanged.connect(self._on_url_changed)
         view.renderProcessTerminated.connect(
@@ -1493,6 +1531,8 @@ class WebEngineTab(browsertab.AbstractTab):
             page.loadFinished.connect(self._on_load_finished)
 
         self.before_load_started.connect(self._on_before_load_started)
+        self.shutting_down.connect(self.abort_questions)
+        self.load_started.connect(self.abort_questions)
 
         # pylint: disable=protected-access
         self.audio._connect_signals()

qutebrowser/browser/webengine/webview.py

@@ -64,6 +64,8 @@ class WebEngineView(QWebEngineView):
         Normally, this would always be the focusProxy().
         However, it sometimes isn't, so we use this as a WORKAROUND for
         https://bugreports.qt.io/browse/QTBUG-68727
+
+        This got introduced in Qt 5.11.0 and fixed in 5.12.0.
         """
         if 'lost-focusproxy' not in objreg.get('args').debug_flags:
             proxy = self.focusProxy()

qutebrowser/browser/webkit/network/networkreply.py

@@ -34,8 +34,7 @@ class FixedDataNetworkReply(QNetworkReply):
 
     """QNetworkReply subclass for fixed data."""
 
-    def __init__(self, request, fileData, mimeType,  # noqa: N803
+    def __init__(self, request, fileData, mimeType, parent=None):  # noqa: N803
-                 parent=None):
         """Constructor.
 
         Args:

qutebrowser/config/configdata.py

@@ -91,15 +91,15 @@ def _parse_yaml_type(
 ) -> configtypes.BaseType:
     if isinstance(node, str):
         # e.g:
-        #  type: Bool
+        #   > type: Bool
         # -> create the type object without any arguments
         type_name = node
         kwargs = {}  # type: typing.MutableMapping[str, typing.Any]
     elif isinstance(node, dict):
         # e.g:
-        #  type:
+        #   > type:
-        #    name: String
+        #   >   name: String
-        #    none_ok: true
+        #   >   none_ok: true
         # -> create the type object and pass arguments
         type_name = node.pop('name')
         kwargs = node
@@ -164,6 +164,7 @@ def _parse_yaml_backends_dict(
         'Qt 5.9.2': qtutils.version_check('5.9.2'),
         'Qt 5.10': qtutils.version_check('5.10'),
         'Qt 5.11': qtutils.version_check('5.11'),
+        'Qt 5.12': qtutils.version_check('5.12'),
     }
     for key in sorted(node.keys()):
         if conditionals[node[key]]:

qutebrowser/config/configdata.yml

@@ -379,7 +379,9 @@ content.developer_extras:
 content.dns_prefetch:
   default: true
   type: Bool
-  backend: QtWebKit
+  backend:
+    QtWebKit: true
+    QtWebEngine: Qt 5.12
   supports_pattern: true
   desc: Try to pre-fetch DNS entries to speed up browsing.
 
@@ -476,46 +478,14 @@ content.headers.user_agent:
       # 'ua_fetch.py'
       # Vim-protip: Place your cursor below this comment and run
       # :r!python scripts/dev/ua_fetch.py
-      - - "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:53.0) Gecko/20100101
-          Firefox/53.0"
-        - Firefox 53.0 Win8.1
-      - - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) Gecko/20100101
-          Firefox/53.0"
-        - Firefox 53.0 Linux
-      - - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:53.0)
-          Gecko/20100101 Firefox/53.0"
-        - Firefox 53.0 MacOSX
-
-      - - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/603.2.4
-          (KHTML, like Gecko) Version/10.1.1 Safari/603.2.4"
-        - Safari Generic MacOSX
-      - - "Mozilla/5.0 (iPad; CPU OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30
-          (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1"
-        - Mobile Safari 10.0 iOS
-
       - - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML,
-          like Gecko) Chrome/58.0.3029.110 Safari/537.36"
+          like Gecko) Chrome/71.0.3578.98 Safari/537.36"
-        - Chrome Generic Win10
+        - Chrome 71.0 Win10
-      - - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36
-          (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
-        - Chrome Generic MacOSX
       - - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like
-          Gecko) Chrome/58.0.3029.110 Safari/537.36"
+          Gecko) Chrome/71.0.3578.98 Safari/537.36"
-        - Chrome Generic Linux
+        - Chrome 71.0 Linux
-
+      - - ""
-      - - "Mozilla/5.0 (compatible; Googlebot/2.1;
+        - Use default QtWebKit/QtWebEngine User-Agent
-          +http://www.google.com/bot.html"
-        - Google Bot
-      - - "Wget/1.16.1 (linux-gnu)"
-        - wget 1.16.1
-      - - "curl/7.40.0"
-        - curl 7.40.0
-      - - "Mozilla/5.0 (Linux; U; Android 7.1.2) AppleWebKit/534.30 (KHTML,
-          like Gecko) Version/4.0 Mobile Safari/534.30"
-        - Mobile Generic Android
-      - - "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like
-          Gecko"
-        - IE 11.0 for Desktop Win7 64-bit
 
   supports_pattern: true
   desc: >-

qutebrowser/javascript/stylesheet.js

@@ -40,6 +40,11 @@ window._qutebrowser.stylesheet = (function() {
     // then move the stylesheet to the end. Partially inspired by Stylus:
     // https://github.com/openstyles/stylus/blob/1.1.4.2/content/apply.js#L235-L355
     function watch_root() {
+        if (!document.documentElement) {
+            root_observer.observe(document, {"childList": true});
+            return;
+        }
+
         if (root_elem !== document.documentElement) {
             root_elem = document.documentElement;
             root_observer.disconnect();
@@ -53,7 +58,8 @@ window._qutebrowser.stylesheet = (function() {
 
     function create_style() {
         let ns = xhtml_ns;
-        if (document.documentElement.namespaceURI === svg_ns) {
+        if (document.documentElement &&
+            document.documentElement.namespaceURI === svg_ns) {
             ns = svg_ns;
         }
         style_elem = document.createElementNS(ns, "style");

qutebrowser/misc/checkpyver.py

@@ -31,7 +31,7 @@ except ImportError:  # pragma: no cover
     try:
         # Python2
         from Tkinter import Tk  # type: ignore
-        import tkMessageBox as messagebox  # type: ignore
+        import tkMessageBox as messagebox  # type: ignore  # noqa: N813
     except ImportError:
         # Some Python without Tk
         Tk = None  # type: ignore

qutebrowser/misc/consolewidget.py

@@ -162,7 +162,7 @@ class ConsoleWidget(QWidget):
         namespace = {
             '__name__': '__console__',
             '__doc__': None,
-            'qApp': QApplication.instance(),
+            'q_app': QApplication.instance(),
             # We use parent as self here because the user "feels" the whole
             # console, not just the line edit.
             'self': parent,

qutebrowser/misc/earlyinit.py

@@ -173,8 +173,10 @@ def check_qt_version():
                               PYQT_VERSION_STR)
     from pkg_resources import parse_version
     from qutebrowser.utils import log
+    parsed_qversion = parse_version(qVersion())
+
     if (QT_VERSION < 0x050701 or PYQT_VERSION < 0x050700 or
-            parse_version(qVersion()) < parse_version('5.7.1')):
+            parsed_qversion < parse_version('5.7.1')):
         text = ("Fatal error: Qt >= 5.7.1 and PyQt >= 5.7 are required, "
                 "but Qt {} / PyQt {} is installed.".format(qt_version(),
                                                            PYQT_VERSION_STR))
@@ -184,6 +186,12 @@ def check_qt_version():
         log.init.warning("Running qutebrowser with Qt 5.8 is untested and "
                          "unsupported!")
 
+    if (parsed_qversion >= parse_version('5.12') and
+            (PYQT_VERSION < 0x050c00 or QT_VERSION < 0x050c00)):
+        log.init.warning("Combining PyQt {} with Qt {} is unsupported! Ensure "
+                         "all versions are newer than 5.12 to avoid potential "
+                         "issues.".format(PYQT_VERSION_STR, qt_version()))
+
 
 def check_ssl_support():
     """Check if SSL support is available."""
@@ -199,19 +207,11 @@ def _check_modules(modules):
 
     for name, text in modules.items():
         try:
-            # https://github.com/pallets/jinja/pull/628
-            # https://bitbucket.org/birkenfeld/pygments-main/issues/1314/
-            # https://github.com/pallets/jinja/issues/646
             # https://bitbucket.org/fdik/pypeg/commits/dd15ca462b532019c0a3be1d39b8ee2f3fa32f4e
-            messages = ['invalid escape sequence',
-                        'Flags not at the start of the expression']
             # pylint: disable=bad-continuation
             with log.ignore_py_warnings(
                 category=DeprecationWarning,
-                message=r'({})'.format('|'.join(messages))
+                message=r'invalid escape sequence'
-            ), log.ignore_py_warnings(
-                category=PendingDeprecationWarning,
-                module='imp'
             ), log.ignore_py_warnings(
                 category=ImportWarning,
                 message=r'Not importing directory .*: missing __init__'

qutebrowser/misc/ipc.py

@@ -65,11 +65,9 @@ def _get_socketname(basedir):
     data_to_hash = '-'.join(parts_to_hash).encode('utf-8')
     md5 = hashlib.md5(data_to_hash).hexdigest()
 
-    target_dir = standarddir.runtime()
+    prefix = 'i-' if utils.is_mac else 'ipc-'
-
+    filename = '{}{}'.format(prefix, md5)
-    parts = ['ipc']
+    return os.path.join(standarddir.runtime(), filename)
-    parts.append(md5)
-    return os.path.join(target_dir, '-'.join(parts))
 
 
 class Error(Exception):

qutebrowser/utils/qtutils.py

@@ -40,7 +40,7 @@ from PyQt5.QtCore import (qVersion, QEventLoop, QDataStream, QByteArray,
 try:
     from PyQt5.QtWebKit import qWebKitVersion
 except ImportError:  # pragma: no cover
-    qWebKitVersion = None  # type: ignore
+    qWebKitVersion = None  # type: ignore  # noqa: N816
 
 
 MAXVALS = {

qutebrowser/utils/utils.py

@@ -30,6 +30,7 @@ import datetime
 import traceback
 import functools
 import contextlib
+import posixpath
 import socket
 import shlex
 import glob
@@ -165,6 +166,9 @@ def read_file(filename, binary=False):
     Return:
         The file contents as string.
     """
+    assert not posixpath.isabs(filename), filename
+    assert os.path.pardir not in filename.split(posixpath.sep), filename
+
     if not binary and filename in _resource_cache:
         return _resource_cache[filename]
 
@@ -655,7 +659,15 @@ def expand_windows_drive(path):
 def yaml_load(f):
     """Wrapper over yaml.load using the C loader if possible."""
     start = datetime.datetime.now()
+
+    # WORKAROUND for https://github.com/yaml/pyyaml/pull/181
+    with log.ignore_py_warnings(
+            category=DeprecationWarning,
+            message=r"Using or importing the ABCs from 'collections' instead "
+            r"of from 'collections\.abc' is deprecated, and in 3\.8 it will "
+            r"stop working"):
         data = yaml.load(f, Loader=YamlLoader)
+
     end = datetime.datetime.now()
 
     delta = (end - start).total_seconds()

qutebrowser/utils/version.py

@@ -42,7 +42,7 @@ from PyQt5.QtWidgets import QApplication
 try:
     from PyQt5.QtWebKit import qWebKitVersion
 except ImportError:  # pragma: no cover
-    qWebKitVersion = None  # type: ignore
+    qWebKitVersion = None  # type: ignore  # noqa: N816
 
 try:
     from PyQt5.QtWebEngineWidgets import QWebEngineProfile
@@ -324,7 +324,7 @@ def _chromium_version():
 
     Qt 5.9:  Chromium 56
     (LTS)    56.0.2924.122 (2017-01-25)
-             5.9.6: Security fixes up to 66.0.3359.170 (2018-05-10)
+             5.9.7: Security fixes up to 69.0.3497.113 (2018-09-27)
 
     Qt 5.10: Chromium 61
              61.0.3163.140 (2017-09-05)
@@ -332,11 +332,14 @@ def _chromium_version():
 
     Qt 5.11: Chromium 65
              65.0.3325.151 (.1: .230) (2018-03-06)
-             5.11.2: Security fixes up to 68.0.3440.75 (2018-07-24)
+             5.11.3: Security fixes up to 70.0.3538.102 (2018-11-09)
 
     Qt 5.12: Chromium 69
-             69.0.3497.128 (~2018-09-17)
+    (LTS)    69.0.3497.113 (2018-09-27)
-             5.12.0: Security fixes up to 70.0.3538.67 (2018-10-16)
+             5.12.1: Security fixes up to 71.0.3578.94 (2018-12-14)
+             5.12.2: Security fixes up to 72.0.3626.96 (2019-02-06)
+
+    Qt 5.13: (in development) Chromium 71 merged, 73 in review.
 
     Also see https://www.chromium.org/developers/calendar
     and https://chromereleases.googleblog.com/

scripts/dev/ci/travis_install.sh

@@ -53,43 +53,13 @@ npm_install() {
     travis_retry npm install -g "$@"
 }
 
-check_pyqt() {
-    python3 <<EOF
-import sys
-from PyQt5.QtCore import QT_VERSION_STR, PYQT_VERSION_STR, qVersion
-try:
-    from PyQt.sip import SIP_VERSION_STR
-except ModuleNotFoundError:
-    from sip import SIP_VERSION_STR
-
-print("Python {}".format(sys.version))
-print("PyQt5 {}".format(PYQT_VERSION_STR))
-print("Qt5 {} (runtime {})".format(QT_VERSION_STR, qVersion()))
-print("sip {}".format(SIP_VERSION_STR))
-EOF
-}
-
 set -e
 
 if [[ -n $DOCKER ]]; then
     exit 0
 elif [[ $TRAVIS_OS_NAME == osx ]]; then
-    # Disable App Nap
-    defaults write NSGlobalDomain NSAppSleepDisabled -bool YES
-
-    curl -LO https://bootstrap.pypa.io/get-pip.py
-    sudo -H python get-pip.py
-
-    brew --version
     brew update
-    brew upgrade python libyaml
+    brew upgrade python
-    brew install qt5 pyqt5
-
-    pip_install -r misc/requirements/requirements-tox.txt
-    python3 -m pip --version
-    tox --version
-    check_pyqt
-    exit 0
 fi
 
 case $TESTENV in

scripts/dev/ci/travis_run.sh

@@ -26,7 +26,8 @@ elif [[ $TESTENV == shellcheck ]]; then
            koalaman/shellcheck:latest "${scripts[@]}"
 else
     args=()
-    [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb' 'tests/unit')
+    # We only run unit tests on macOS because it's quite slow.
+    [[ $TRAVIS_OS_NAME == osx ]] && args+=('--qute-bdd-webengine' '--no-xvfb' 'tests/unit')
 
     # WORKAROUND for unknown crash inside swrast_dri.so
     # See https://github.com/qutebrowser/qutebrowser/pull/4218#issuecomment-421931770

scripts/dev/ua_fetch.py

@@ -75,42 +75,17 @@ def filter_list(complete_list, browsers):
     return table
 
 
-def add_diversity(table):
-    """Insert a few additional entries for diversity into the dict.
-
-    (as returned by filter_list())
-    """
-    table["Obscure"] = [
-        ('Mozilla/5.0 (compatible; Googlebot/2.1; '
-         '+http://www.google.com/bot.html',
-         "Google Bot"),
-        ('Wget/1.16.1 (linux-gnu)',
-         "wget 1.16.1"),
-        ('curl/7.40.0',
-         "curl 7.40.0"),
-        ('Mozilla/5.0 (Linux; U; Android 7.1.2) AppleWebKit/534.30 '
-         '(KHTML, like Gecko) Version/4.0 Mobile Safari/534.30',
-         "Mobile Generic Android"),
-        ('Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like '
-         'Gecko',
-         "IE 11.0 for Desktop  Win7 64-bit"),
-    ]
-    return table
-
-
 def main():
     """Generate user agent code."""
     fetched = fetch()
     lut = {
-        "Firefox": {"Win", "MacOSX", "Linux", "Android"},
+        "Chrome": {"Win10", "Linux"},
-        "Chrome": {"Win", "MacOSX", "Linux"},
-        "Safari": {"MacOSX", "iOS"}
     }
     filtered = filter_list(fetched, lut)
-    filtered = add_diversity(filtered)
+    filtered["empty"] = [('', "Use default QtWebKit/QtWebEngine User-Agent")]
 
     tab = "  "
-    for browser in ["Firefox", "Safari", "Chrome", "Obscure"]:
+    for browser in ["Chrome", "empty"]:
         for it in filtered[browser]:
             print('{}- - "{}"'.format(3 * tab, it[0]))
             desc = it[1].replace('\xa0', ' ').replace('  ', ' ')

tests/end2end/features/conftest.py

@@ -20,6 +20,7 @@
 """Steps for bdd-like tests."""
 
 import os
+import os.path
 import re
 import sys
 import time
@@ -27,11 +28,13 @@ import json
 import logging
 import collections
 import textwrap
+import subprocess
 
 import pytest
 import pytest_bdd as bdd
 
-from qutebrowser.utils import log, utils
+import qutebrowser
+from qutebrowser.utils import log, utils, docutils
 from qutebrowser.browser import pdfjs
 from helpers import utils as testutils
 
@@ -382,6 +385,32 @@ def clear_ssl_errors(request, quteproc):
         quteproc.send_cmd(':debug-clear-ssl-errors')
 
 
+@bdd.when("the documentation is up to date")
+def update_documentation():
+    """Update the docs before testing :help."""
+    base_path = os.path.dirname(os.path.abspath(qutebrowser.__file__))
+    doc_path = os.path.join(base_path, 'html', 'doc')
+    script_path = os.path.join(base_path, '..', 'scripts')
+
+    try:
+        os.mkdir(doc_path)
+    except FileExistsError:
+        pass
+
+    files = os.listdir(doc_path)
+    if files and all(docutils.docs_up_to_date(p) for p in files):
+        return
+
+    try:
+        subprocess.run(['asciidoc'], stdout=subprocess.DEVNULL,
+                       stderr=subprocess.DEVNULL)
+    except OSError:
+        pytest.skip("Docs outdated and asciidoc unavailable!")
+
+    update_script = os.path.join(script_path, 'asciidoc2html.py')
+    subprocess.run([sys.executable, update_script])
+
+
 ## Then
 
 

tests/end2end/features/misc.feature

@@ -331,19 +331,16 @@ Feature: Various utility commands.
         When I set content.headers.do_not_track to true
         And I open headers
         Then the header Dnt should be set to 1
-        And the header X-Do-Not-Track should be set to 1
 
     Scenario: DNT header (off)
         When I set content.headers.do_not_track to false
         And I open headers
         Then the header Dnt should be set to 0
-        And the header X-Do-Not-Track should be set to 0
 
     Scenario: DNT header (unset)
         When I set content.headers.do_not_track to <empty>
         And I open headers
         Then the header Dnt should be set to <unset>
-        And the header X-Do-Not-Track should be set to <unset>
 
     Scenario: Accept-Language header
         When I set content.headers.accept_language to en,de

tests/end2end/features/navigate.feature

@@ -24,7 +24,8 @@ Feature: Using :navigate
         Then data/navigate should be loaded
 
     Scenario: Navigating up in qute://help/
-        When I open qute://help/commands.html
+        When the documentation is up to date
+        And I open qute://help/commands.html
         And I run :navigate up
         Then qute://help/ should be loaded
 

tests/end2end/features/qutescheme.feature

@@ -8,7 +8,8 @@ Feature: Special qute:// pages
     # :help
 
     Scenario: :help without topic
-        When I run :tab-only
+        When the documentation is up to date
+        And I run :tab-only
         And I run :help
         And I wait until qute://help/index.html is loaded
         Then the following tabs should be open:
@@ -39,7 +40,8 @@ Feature: Special qute:// pages
             - qute://help/settings.html#editor.command (active)
 
     Scenario: :help with -t
-        When I run :tab-only
+        When the documentation is up to date
+        And I run :tab-only
         And I run :help -t
         And I wait until qute://help/index.html is loaded
         Then the following tabs should be open:
@@ -140,29 +142,25 @@ Feature: Special qute:// pages
         And I press the key "<Tab>"
         Then "Invalid value 'foo' *" should be logged
 
-    @qtwebkit_skip
+    Scenario: qute://settings CSRF via img
-    Scenario: qute://settings CSRF via img (webengine)
         When I open data/misc/qutescheme_csrf.html
         And I run :click-element id via-img
-        Then "Blocking malicious request from http://localhost:*/data/misc/qutescheme_csrf.html to qute://settings/set?*" should be logged
+        Then the img request should be blocked
 
-    @qtwebkit_skip
+    Scenario: qute://settings CSRF via link
-    Scenario: qute://settings CSRF via link (webengine)
         When I open data/misc/qutescheme_csrf.html
         And I run :click-element id via-link
-        Then "Blocking malicious request from qute://settings/set?* to qute://settings/set?*" should be logged
+        Then the link request should be blocked
 
-    @qtwebkit_skip
+    Scenario: qute://settings CSRF via redirect
-    Scenario: qute://settings CSRF via redirect (webengine)
         When I open data/misc/qutescheme_csrf.html
         And I run :click-element id via-redirect
-        Then "Blocking malicious request from qute://settings/set?* to qute://settings/set?*" should be logged
+        Then the redirect request should be blocked
 
-    @qtwebkit_skip
+    Scenario: qute://settings CSRF via form
-    Scenario: qute://settings CSRF via form (webengine)
         When I open data/misc/qutescheme_csrf.html
         And I run :click-element id via-form
-        Then "Blocking malicious request from qute://settings/set?* to qute://settings/set?*" should be logged
+        Then the form request should be blocked
 
     @qtwebkit_skip
     Scenario: qute://settings CSRF token (webengine)
@@ -171,32 +169,6 @@ Feature: Special qute:// pages
         Then "RequestDeniedError while handling qute://* URL" should be logged
         And the error "Invalid CSRF token for qute://settings!" should be shown
 
-    @qtwebengine_skip
-    Scenario: qute://settings CSRF via img (webkit)
-        When I open data/misc/qutescheme_csrf.html
-        And I run :click-element id via-img
-        Then "Blocking malicious request from http://localhost:*/data/misc/qutescheme_csrf.html to qute://settings/set?*" should be logged
-
-    @qtwebengine_skip
-    Scenario: qute://settings CSRF via link (webkit)
-        When I open data/misc/qutescheme_csrf.html
-        And I run :click-element id via-link
-        Then "Blocking malicious request from http://localhost:*/data/misc/qutescheme_csrf.html to qute://settings/set?*" should be logged
-        And "Error while loading qute://settings/set?*: Invalid qute://settings request" should be logged
-
-    @qtwebengine_skip
-    Scenario: qute://settings CSRF via redirect (webkit)
-        When I open data/misc/qutescheme_csrf.html
-        And I run :click-element id via-redirect
-        Then "Blocking malicious request from http://localhost:*/data/misc/qutescheme_csrf.html to qute://settings/set?*" should be logged
-        And "Error while loading qute://settings/set?*: Invalid qute://settings request" should be logged
-
-    @qtwebengine_skip
-    Scenario: qute://settings CSRF via form (webkit)
-        When I open data/misc/qutescheme_csrf.html
-        And I run :click-element id via-form
-        Then "Error while loading qute://settings/set?*: Unsupported request type" should be logged
-
     # pdfjs support
 
     Scenario: pdfjs is used for pdf files

tests/end2end/features/sessions.feature

@@ -228,7 +228,7 @@ Feature: Saving and loading sessions
             url: http://localhost:*/data/hello.txt
 
   # Seems like that bug is fixed upstream in QtWebEngine
-  @qtwebkit_skip @flaky
+  @skip  # Too flaky
   Scenario: Saving a session with a page using history.replaceState() and navigating away
     When I open data/sessions/history_replace_state.html without waiting
     And I wait for "* Called history.replaceState" in the log

tests/end2end/features/test_qutescheme_bdd.py

@@ -17,40 +17,58 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-import sys
-import os.path
-import subprocess
-
-import pytest
 import pytest_bdd as bdd
 
-import qutebrowser
+from qutebrowser.utils import qtutils
-from qutebrowser.utils import docutils
+
 
 bdd.scenarios('qutescheme.feature')
 
 
-@bdd.when("the documentation is up to date")
+@bdd.then(bdd.parsers.parse("the {kind} request should be blocked"))
-def update_documentation():
+def request_blocked(request, quteproc, kind):
-    """Update the docs before testing :help."""
+    blocking_set_msg = (
-    base_path = os.path.dirname(os.path.abspath(qutebrowser.__file__))
+        "Blocking malicious request from qute://settings/set?* to "
-    doc_path = os.path.join(base_path, 'html', 'doc')
+        "qute://settings/set?*")
-    script_path = os.path.join(base_path, '..', 'scripts')
+    blocking_csrf_msg = (
+        "Blocking malicious request from "
+        "http://localhost:*/data/misc/qutescheme_csrf.html to "
+        "qute://settings/set?*")
+    blocking_js_msg = (
+        "[http://localhost:*/data/misc/qutescheme_csrf.html:0] Not allowed to "
+        "load local resource: qute://settings/set?*"
+    )
 
-    try:
+    webkit_error_invalid = (
-        os.mkdir(doc_path)
+        "Error while loading qute://settings/set?*: Invalid qute://settings "
-    except FileExistsError:
+        "request")
-        pass
+    webkit_error_unsupported = (
+        "Error while loading qute://settings/set?*: Unsupported request type")
 
-    files = os.listdir(doc_path)
+    if request.config.webengine and qtutils.version_check('5.12'):
-    if files and all(docutils.docs_up_to_date(p) for p in files):
+        # On Qt 5.12, we mark qute:// as a local scheme, causing most requests
-        return
+        # being blocked by Chromium internally (logging to the JS console).
+        expected_messages = {
+            'img': [blocking_js_msg],
+            'link': [blocking_js_msg],
+            'redirect': [blocking_set_msg],
+            'form': [blocking_js_msg],
+        }
+    elif request.config.webengine:
+        expected_messages = {
+            'img': [blocking_csrf_msg],
+            'link': [blocking_set_msg],
+            'redirect': [blocking_set_msg],
+            'form': [blocking_set_msg],
+        }
+    else:  # QtWebKit
+        expected_messages = {
+            'img': [blocking_csrf_msg],
+            'link': [blocking_csrf_msg, webkit_error_invalid],
+            'redirect': [blocking_csrf_msg, webkit_error_invalid],
+            'form': [webkit_error_unsupported],
+        }
 
-    try:
+    for pattern in expected_messages[kind]:
-        subprocess.run(['asciidoc'], stdout=subprocess.DEVNULL,
+        msg = quteproc.wait_for(message=pattern)
-                       stderr=subprocess.DEVNULL)
+        msg.expected = True
-    except OSError:
-        pytest.skip("Docs outdated and asciidoc unavailable!")
-
-    update_script = os.path.join(script_path, 'asciidoc2html.py')
-    subprocess.run([sys.executable, update_script])

tests/unit/browser/test_shared.py

@@ -26,18 +26,15 @@ from qutebrowser.browser import shared
 
 @pytest.mark.parametrize('dnt, accept_language, custom_headers, expected', [
     # DNT
-    (True, None, {}, {b'DNT': b'1', b'X-Do-Not-Track': b'1'}),
+    (True, None, {}, {b'DNT': b'1'}),
-    (False, None, {}, {b'DNT': b'0', b'X-Do-Not-Track': b'0'}),
+    (False, None, {}, {b'DNT': b'0'}),
     (None, None, {}, {}),
     # Accept-Language
-    (False, 'de, en', {}, {b'DNT': b'0', b'X-Do-Not-Track': b'0',
+    (False, 'de, en', {}, {b'DNT': b'0', b'Accept-Language': b'de, en'}),
-                           b'Accept-Language': b'de, en'}),
     # Custom headers
-    (False, None, {'X-Qute': 'yes'}, {b'DNT': b'0', b'X-Do-Not-Track': b'0',
+    (False, None, {'X-Qute': 'yes'}, {b'DNT': b'0', b'X-Qute': b'yes'}),
-                                      b'X-Qute': b'yes'}),
     # Mixed
     (False, 'de, en', {'X-Qute': 'yes'}, {b'DNT': b'0',
-                                          b'X-Do-Not-Track': b'0',
                                           b'Accept-Language': b'de, en',
                                           b'X-Qute': b'yes'}),
 ])

tests/unit/browser/webengine/test_webenginedownloads.py

@@ -30,6 +30,8 @@ from helpers import utils
 @pytest.mark.parametrize('path, expected', [
     (os.path.join('subfolder', 'foo'), 'foo'),
     ('foo(1)', 'foo'),
+    ('foo (1)', 'foo'),
+    ('foo - 1970-01-01T00:00:00.000Z', 'foo'),
     ('foo(a)', 'foo(a)'),
     ('foo1', 'foo1'),
     pytest.param('foo%20bar', 'foo bar', marks=utils.qt58),

tests/unit/completion/test_completionmodel.py

@@ -41,7 +41,7 @@ def test_first_last_item(counts):
         cat = mock.Mock(spec=['layoutChanged', 'layoutAboutToBeChanged'])
         cat.rowCount = mock.Mock(return_value=c, spec=[])
         model.add_category(cat)
-    data = [i for i, rowCount in enumerate(counts) if rowCount > 0]
+    data = [i for i, row_count in enumerate(counts) if row_count > 0]
     if not data:
         # with no items, first and last should be an invalid index
         assert not model.first_item().isValid()

tests/unit/components/test_adblock.py

@@ -30,7 +30,7 @@ from PyQt5.QtCore import QUrl
 
 from qutebrowser.components import adblock
 from qutebrowser.utils import urlmatch
-from tests.helpers import utils
+from helpers import utils
 
 
 pytestmark = pytest.mark.usefixtures('qapp')

tests/unit/config/test_configtypes.py

@@ -38,7 +38,7 @@ from qutebrowser.config import configtypes, configexc, configutils
 from qutebrowser.utils import debug, utils, qtutils, urlmatch
 from qutebrowser.browser.network import pac
 from qutebrowser.keyinput import keyutils
-from tests.helpers import utils as testutils
+from helpers import utils as testutils
 
 
 class Font(QFont):

tests/unit/keyinput/test_keyutils.py

@@ -26,7 +26,7 @@ from PyQt5.QtCore import Qt, QEvent, pyqtSignal
 from PyQt5.QtGui import QKeyEvent, QKeySequence
 from PyQt5.QtWidgets import QWidget
 
-from tests.unit.keyinput import key_data
+from unit.keyinput import key_data
 from qutebrowser.keyinput import keyutils
 from qutebrowser.utils import utils
 

tests/unit/misc/test_ipc.py

@@ -34,7 +34,7 @@ from PyQt5.QtTest import QSignalSpy
 
 import qutebrowser
 from qutebrowser.misc import ipc
-from qutebrowser.utils import standarddir, utils, qtutils
+from qutebrowser.utils import standarddir, utils
 from helpers import stubs
 
 
@@ -98,7 +98,7 @@ class FakeSocket(QObject):
         _connect_successful: The value returned for waitForConnected().
     """
 
-    readyRead = pyqtSignal()
+    readyRead = pyqtSignal()  # noqa: N815
     disconnected = pyqtSignal()
 
     def __init__(self, *, error=QLocalSocket.UnknownSocketError, state=None,
@@ -177,11 +177,6 @@ def md5(inp):
 
 class TestSocketName:
 
-    POSIX_TESTS = [
-        (None, 'ipc-{}'.format(md5('testusername'))),
-        ('/x', 'ipc-{}'.format(md5('testusername-/x'))),
-    ]
-
     WINDOWS_TESTS = [
         (None, 'qutebrowser-testusername'),
         ('/x', 'qutebrowser-testusername-{}'.format(md5('/x'))),
@@ -203,7 +198,10 @@ class TestSocketName:
         assert socketname == expected
 
     @pytest.mark.mac
-    @pytest.mark.parametrize('basedir, expected', POSIX_TESTS)
+    @pytest.mark.parametrize('basedir, expected', [
+        (None, 'i-{}'.format(md5('testusername'))),
+        ('/x', 'i-{}'.format(md5('testusername-/x'))),
+    ])
     def test_mac(self, basedir, expected):
         socketname = ipc._get_socketname(basedir)
         parts = socketname.split(os.sep)
@@ -211,7 +209,10 @@ class TestSocketName:
         assert parts[-1] == expected
 
     @pytest.mark.linux
-    @pytest.mark.parametrize('basedir, expected', POSIX_TESTS)
+    @pytest.mark.parametrize('basedir, expected', [
+        (None, 'ipc-{}'.format(md5('testusername'))),
+        ('/x', 'ipc-{}'.format(md5('testusername-/x'))),
+    ])
     def test_linux(self, basedir, fake_runtime_dir, expected):
         socketname = ipc._get_socketname(basedir)
         expected_path = str(fake_runtime_dir / 'qutebrowser' / expected)
@@ -630,8 +631,6 @@ class TestSendOrListen:
         assert ret_client is None
 
     @pytest.mark.posix(reason="Unneeded on Windows")
-    @pytest.mark.xfail(qtutils.version_check('5.12', compiled=False) and
-                       utils.is_mac, reason="Broken, see #4471")
     def test_correct_socket_name(self, args):
         server = ipc.send_or_listen(args)
         expected_dir = ipc._get_socketname(args.basedir)

tests/unit/test_app.py

@@ -30,7 +30,7 @@ def test_on_focus_changed_issue1484(monkeypatch, qapp, caplog):
     For some reason, Qt sometimes calls on_focus_changed() with a QBuffer as
     argument. Let's make sure we handle that gracefully.
     """
-    monkeypatch.setattr(app, 'qApp', qapp)
+    monkeypatch.setattr(app, 'q_app', qapp)
 
     buf = QBuffer()
     app.on_focus_changed(buf, buf)

tox.ini

@@ -13,8 +13,8 @@ skipsdist = true
 setenv =
     QT_QPA_PLATFORM_PLUGIN_PATH={envdir}/Lib/site-packages/PyQt5/plugins/platforms
     PYTEST_QT_API=pyqt5
-    pyqt{,56,571,59,510,511}: LINK_PYQT_SKIP=true
+    pyqt{,56,571,59,510,511,512}: LINK_PYQT_SKIP=true
-    pyqt{,56,571,59,510,511}: QUTE_BDD_WEBENGINE=true
+    pyqt{,56,571,59,510,511,512}: QUTE_BDD_WEBENGINE=true
     cov: PYTEST_ADDOPTS=--cov --cov-report xml --cov-report=html --cov-report=
 passenv = PYTHON DISPLAY XAUTHORITY HOME USERNAME USER CI TRAVIS XDG_* QUTE_* DOCKER QT_QUICK_BACKEND
 basepython =
@@ -29,6 +29,7 @@ deps =
     pyqt59: PyQt5==5.9.2
     pyqt510: PyQt5==5.10.1
     pyqt511: PyQt5==5.11.3
+    pyqt512: PyQtWebEngine==5.12
 commands =
     {envpython} scripts/link_pyqt.py --tox {envdir}
     {envpython} -bb -m pytest {posargs:tests}