Merge branch 'master' into new-config

This pulls the travis changes to drop the old Qt 5.2 environment.

.travis.yml

@@ -1,12 +1,11 @@
-sudo: required
+sudo: false
 dist: trusty
-language: generic
+language: python
 group: edge
+python: 3.6
 
 matrix:
   include:
-    - os: linux
-      env: TESTENV=py34-cov
     - os: linux
       env: DOCKER=debian-jessie
       services: docker
@@ -20,36 +19,32 @@ matrix:
       env: DOCKER=ubuntu-xenial
       services: docker
     - os: linux
-      language: python
-      python: 3.6
       env: TESTENV=py36-pyqt571
     - os: linux
-      language: python
-      python: 3.6
       env: TESTENV=py36-pyqt58
     - os: linux
-      language: python
       python: 3.5
       env: TESTENV=py35-pyqt59
     - os: linux
-      language: python
-      python: 3.6
-      env: TESTENV=py36-pyqt59
+      env: TESTENV=py36-pyqt59-cov
     - os: osx
-      env: TESTENV=py36 OSX=elcapitan
-      osx_image: xcode7.3
+      env: TESTENV=py36 OSX=sierra
+      osx_image: xcode8.3
+      language: generic
     # https://github.com/qutebrowser/qutebrowser/issues/2013
     # - os: osx
     #   env: TESTENV=py35 OSX=yosemite
     #   osx_image: xcode6.4
     - os: linux
       env: TESTENV=pylint PYTHON=python3.6
-      language: python
-      python: 3.6
     - os: linux
       env: TESTENV=flake8
     - os: linux
       env: TESTENV=docs
+      addons:
+         apt:
+           packages:
+           - asciidoc
     - os: linux
       env: TESTENV=vulture
     - os: linux
@@ -60,10 +55,9 @@ matrix:
       env: TESTENV=check-manifest
     - os: linux
       env: TESTENV=eslint
-  allow_failures:
-    - os: osx
-      env: TESTENV=py36 OSX=elcapitan
-      osx_image: xcode7.3
+      language: node_js
+      python: null
+      node_js: node
   fast_finish: true
 
 cache:
@@ -71,10 +65,6 @@ cache:
     - $HOME/.cache/pip
     - $HOME/build/qutebrowser/qutebrowser/.cache
 
-before_install:
-  # We need to do this so we pick up the system-wide python properly
-  - 'export PATH="/usr/bin:$PATH"'
-
 install:
   - bash scripts/dev/ci/travis_install.sh
   - ulimit -c unlimited

pytest.ini

@@ -45,10 +45,6 @@ qt_log_ignore =
     ^QWaitCondition: Destroyed while threads are still waiting
     ^QXcbXSettings::QXcbXSettings\(QXcbScreen\*\) Failed to get selection owner for XSETTINGS_S atom
     ^QStandardPaths: XDG_RUNTIME_DIR not set, defaulting to .*
-    ^QGeoclueMaster error creating GeoclueMasterClient\.
-    ^Geoclue error: Process org\.freedesktop\.Geoclue\.Master exited with status 127
-    ^Failed to create Geoclue client interface. Geoclue error: org\.freedesktop\.DBus\.Error\.Disconnected
-    ^QDBusConnection: name 'org.freedesktop.Geoclue.Master' had owner '' but we thought it was ':1.1'
     ^QObject::connect: Cannot connect \(null\)::stateChanged\(QNetworkSession::State\) to QNetworkReplyHttpImpl::_q_networkSessionStateChanged\(QNetworkSession::State\)
     ^QXcbClipboard: Cannot transfer data, no data available
     ^load glyph failed

qutebrowser/browser/history.py

@@ -173,9 +173,7 @@ class WebHistory(sql.SqlTable):
                       (hidden in completion)
             atime: Override the atime used to add the entry
         """
-        if not url.isValid():  # pragma: no cover
-            # the no cover pragma is a WORKAROUND for this not being covered in
-            # old Qt versions.
+        if not url.isValid():
             log.misc.warning("Ignoring invalid URL being added to history")
             return
 
@@ -320,6 +318,6 @@ def init(parent=None):
     history = WebHistory(parent=parent)
     objreg.register('web-history', history)
 
-    if objects.backend == usertypes.Backend.QtWebKit:
+    if objects.backend == usertypes.Backend.QtWebKit:  # pragma: no cover
         from qutebrowser.browser.webkit import webkithistory
         webkithistory.init(history)

qutebrowser/browser/webkit/certificateerror.py

@@ -41,7 +41,7 @@ class CertificateErrorWrapper(usertypes.AbstractCertificateErrorWrapper):
         try:
             # Qt >= 5.4
             return hash(self._error)
-        except TypeError:  # pragma: no cover
+        except TypeError:
             return hash((self._error.certificate().toDer(),
                          self._error.error()))
 

qutebrowser/mainwindow/statusbar/backforward.py

@@ -30,7 +30,6 @@ class Backforward(textbase.TextBase):
         """Called on URL changes."""
         tab = tabs.currentWidget()
         if tab is None:  # pragma: no cover
-            # WORKAROUND: Doesn't get tested on older PyQt
             self.setText('')
             self.hide()
             return

qutebrowser/misc/miscwidgets.py

@@ -246,8 +246,7 @@ class WrapperLayout(QLayout):
     def sizeHint(self):
         return self._widget.sizeHint()
 
-    def itemAt(self, _index):  # pragma: no cover
-        # For some reason this sometimes gets called by Qt.
+    def itemAt(self, _index):
         return None
 
     def takeAt(self, _index):

qutebrowser/utils/urlutils.py

@@ -604,7 +604,7 @@ def safe_display_string(qurl):
         raise InvalidUrlError(qurl)
 
     host = qurl.host(QUrl.FullyEncoded)
-    if '..' in host:
+    if '..' in host:  # pragma: no cover
         # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-60364
         return '(unparseable URL!) {}'.format(qurl.toDisplayString())
 

scripts/dev/check_coverage.py

@@ -53,8 +53,6 @@ PERFECT_FILES = [
         'browser/webkit/cookies.py'),
     ('tests/unit/browser/test_history.py',
         'browser/history.py'),
-    ('tests/unit/browser/test_history.py',
-        'browser/webkit/webkithistory.py'),
     ('tests/unit/browser/webkit/http/test_http.py',
         'browser/webkit/http.py'),
     ('tests/unit/browser/webkit/http/test_content_disposition.py',
@@ -73,7 +71,7 @@ PERFECT_FILES = [
     ('tests/unit/browser/test_signalfilter.py',
         'browser/signalfilter.py'),
     (None,
-        'browser/webkit/certificateerror.py'),
+        'browser/webengine/certificateerror.py'),
     # ('tests/unit/browser/test_tab.py',
     #     'browser/tab.py'),
 
@@ -272,8 +270,8 @@ def main_check():
         subprocess.check_call([sys.executable, '-m', 'coverage', 'report',
                                '--show-missing', '--include', filters])
         print()
-        print("To debug this, run 'tox -e py35-cov' (or py34-cov) locally and "
-              "check htmlcov/index.html")
+        print("To debug this, run 'tox -e py36-pyqt59-cov' "
+              "(or py35-pyqt59-cov) locally and check htmlcov/index.html")
         print("or check https://codecov.io/github/qutebrowser/qutebrowser")
         print()
 

scripts/dev/ci/travis_install.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # vim: ft=sh fileencoding=utf-8 sts=4 sw=4 et:
 
 # Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
@@ -42,37 +43,19 @@ travis_retry() {
     return $result
 }
 
-apt_install() {
-    sudo tee /etc/apt/sources.list <<EOF
-deb http://us.archive.ubuntu.com/ubuntu/ trusty main
-deb http://us.archive.ubuntu.com/ubuntu/ trusty-security main
-deb http://us.archive.ubuntu.com/ubuntu/ trusty-updates main
-EOF
-    sudo rm -rf /etc/apt/sources.list.d
-    travis_retry sudo apt-get -y -q update
-    travis_retry sudo apt-get -y -q install --no-install-recommends "$@"
-}
-
 brew_install() {
     brew update
     brew install "$@"
 }
 
 pip_install() {
-    # this uses python2
-    travis_retry sudo -H python -m pip install "$@"
+    travis_retry python3 -m pip install "$@"
 }
 
 npm_install() {
     # Make sure npm is up-to-date first
-    travis_retry sudo npm install -g npm
-    travis_retry sudo npm install -g "$@"
-}
-
-install_node() {
-    curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash -
-    travis_retry sudo apt-get -y -q update
-    travis_retry sudo apt-get -y -q install --no-install-recommends nodejs
+    travis_retry npm install -g npm
+    travis_retry npm install -g "$@"
 }
 
 check_pyqt() {
@@ -103,55 +86,18 @@ elif [[ $TRAVIS_OS_NAME == osx ]]; then
     brew_install python3 qt5 pyqt5
 
     pip_install -r misc/requirements/requirements-tox.txt
-    pip --version
+    python3 -m pip --version
     tox --version
     check_pyqt
     exit 0
 fi
 
-pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtquick python3-pyqt5.qtwebkit python3-pyqt5.qtsql libqt5sql5-sqlite"
-
-pip_install pip
-pip_install -r misc/requirements/requirements-tox.txt
-
-pip --version
-tox --version
-
 case $TESTENV in
-    py34-cov)
-        pip_install -r misc/requirements/requirements-codecov.txt
-        apt_install xvfb $pyqt_pkgs libpython3.4-dev gdb apport libqt5webkit5-dbg python3-pyqt5-dbg python3-pyqt5.qtquick-dbg python3-pyqt5.qtwebkit-dbg python3-dbg
-        check_pyqt
-        ;;
-    py3*-pyqt*)
-        apt_install xvfb geoclue gdb apport
-        ;;
-    pylint|vulture)
-        apt_install $pyqt_pkgs libpython3.4-dev
-        check_pyqt
-        ;;
-    flake8)
-        apt_install libpython3.4-dev
-        ;;
-    docs)
-        apt_install $pyqt_pkgs asciidoc libpython3.4-dev
-        asciidoc --version
-        check_pyqt
-        ;;
-    misc)
-        apt_install libpython3.4-dev
-        ;;
-    pyroma|check-manifest)
-        ;;
     eslint)
-        install_node
-        echo "node: $(node --version)"
-        echo "npm: $(npm --version)"
         npm_install eslint
-        echo "eslint: $(eslint --version)"
         ;;
     *)
-        echo "Unknown testenv $TESTENV!" >&2
-        exit 1
+        pip_install pip
+        pip_install -r misc/requirements/requirements-tox.txt
         ;;
 esac

scripts/dev/ci/travis_run.sh

@@ -1,10 +1,14 @@
 #!/bin/bash
 
 if [[ $DOCKER ]]; then
-    docker run --privileged -v $PWD:/outside -e QUTE_BDD_WEBENGINE=$QUTE_BDD_WEBENGINE -e DOCKER=$DOCKER -e CI=$CI qutebrowser/travis:$DOCKER
+    docker run --privileged -v "$PWD:/outside" -e "QUTE_BDD_WEBENGINE=$QUTE_BDD_WEBENGINE" -e "DOCKER=$DOCKER" -e "CI=$CI" "qutebrowser/travis:$DOCKER"
+elif [[ $TESTENV == eslint ]]; then
+    # Can't run this via tox as we can't easily install tox in the javascript travis env
+    cd qutebrowser/javascript || exit 1
+    eslint --color .
 else
     args=()
     [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb')
 
-    tox -e $TESTENV -- "${args[@]}"
+    tox -e "$TESTENV" -- "${args[@]}"
 fi

tests/end2end/data/userscripts/stdinclose.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
 # Copyright 2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>

tests/end2end/features/completion.feature

@@ -50,6 +50,8 @@ Feature: Using completion
         When I run :set-cmd-text -s :bind X
         Then the completion model should be bind
 
+    # See #2956
+    @qtwebengine_mac_xfail
     Scenario: Using session completion
         Given I open data/hello.txt
         And I run :session-save hello

tests/end2end/features/prompts.feature

@@ -219,22 +219,6 @@ Feature: Prompts
         And I run :click-element id button
         Then the javascript message "geolocation permission denied" should be logged
 
-    @ci @not_mac @qt!=5.8
-    Scenario: Always accepting geolocation
-        When I set content.geolocation to true
-        And I open data/prompt/geolocation.html in a new tab
-        And I run :click-element id button
-        Then the javascript message "geolocation permission denied" should not be logged
-
-    @ci @not_mac @qt!=5.8
-    Scenario: geolocation with ask -> true
-        When I set content.geolocation to ask
-        And I open data/prompt/geolocation.html in a new tab
-        And I run :click-element id button
-        And I wait for a prompt
-        And I run :prompt-accept yes
-        Then the javascript message "geolocation permission denied" should not be logged
-
     Scenario: geolocation with ask -> false
         When I set content.geolocation to ask
         And I open data/prompt/geolocation.html in a new tab

tests/end2end/features/yankpaste.feature

@@ -291,7 +291,6 @@ Feature: Yanking and pasting.
         # Compare
         Then the javascript message "textarea contents: onHello worlde two three four" should be logged
 
-    @qtwebengine_mac_xfail
     Scenario: Inserting text into a text field with undo
         When I set content.javascript.log to info
         And I open data/paste_primary.html

tests/end2end/fixtures/quteprocess.py

@@ -66,6 +66,21 @@ def is_ignored_lowlevel_message(message):
         return True
     elif message == 'getrlimit(RLIMIT_NOFILE) failed':
         return True
+    # Travis CI containers don't have a /etc/machine-id
+    elif message.endswith('D-Bus library appears to be incorrectly set up; '
+                          'failed to read machine uuid: Failed to open '
+                          '"/etc/machine-id": No such file or directory'):
+        return True
+    elif message == ('See the manual page for dbus-uuidgen to correct this '
+                     'issue.'):
+        return True
+    # Travis CI macOS:
+    # 2017-09-11 07:32:56.191 QtWebEngineProcess[5455:28501] Couldn't set
+    # selectedTextBackgroundColor from default ()
+
+    elif message.endswith("Couldn't set selectedTextBackgroundColor from "
+                          "default ()"):
+        return True
     return False
 
 
@@ -127,6 +142,11 @@ def is_ignored_chromium_message(line):
         # [5947:5947:0605/192837.856931:ERROR:render_process_impl.cc(112)]
         # WebFrame LEAKED 1 TIMES
         'WebFrame LEAKED 1 TIMES',
+
+        # macOS on Travis
+        # [5140:5379:0911/063441.239771:ERROR:mach_port_broker.mm(175)]
+        # Unknown process 5176 is sending Mach IPC messages!
+        'Unknown process * is sending Mach IPC messages!',
     ]
     return any(testutils.pattern_match(pattern=pattern, value=message)
                for pattern in ignored_messages)

tests/end2end/test_invocations.py

@@ -175,15 +175,16 @@ def test_version(request):
     proc.start(sys.executable, args)
     ok = proc.waitForStarted(2000)
     assert ok
-    ok = proc.waitForFinished(2000)
-    assert ok
-    assert proc.exitStatus() == QProcess.NormalExit
+    ok = proc.waitForFinished(10000)
 
     stdout = bytes(proc.readAllStandardOutput()).decode('utf-8')
     print(stdout)
     stderr = bytes(proc.readAllStandardError()).decode('utf-8')
     print(stderr)
 
+    assert ok
+    assert proc.exitStatus() == QProcess.NormalExit
+
     assert re.search(r'^qutebrowser\s+v\d+(\.\d+)', stdout) is not None
 
 

tests/unit/utils/usertypes/test_question.py

@@ -89,3 +89,12 @@ def test_abort_typeerror(question, qtbot, mocker, caplog):
     with caplog.at_level(logging.ERROR, 'misc'):
         question.abort()
     assert caplog.records[0].message == 'Error while aborting question'
+
+
+def test_abort_twice(question, qtbot):
+    """Abort a question twice."""
+    with qtbot.wait_signal(question.aborted):
+        question.abort()
+    assert question.is_aborted
+    with qtbot.assert_not_emitted(question.aborted):
+        question.abort()

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py36-cov,misc,vulture,flake8,pylint,pyroma,check-manifest,eslint
+envlist = py36-pyqt59-cov,misc,vulture,flake8,pylint,pyroma,check-manifest,eslint
 distshare = {toxworkdir}
 skipsdist = true
 
@@ -21,39 +21,6 @@ commands =
     {envpython} scripts/link_pyqt.py --tox {envdir}
     {envpython} -bb -m pytest {posargs:tests}
 
-
-# test envs with coverage
-
-[testenv:py36-cov]
-basepython = python3.6
-setenv = {[testenv]setenv}
-passenv = {[testenv]passenv}
-deps = {[testenv]deps}
-commands =
-    {envpython} scripts/link_pyqt.py --tox {envdir}
-    {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests}
-    {envpython} scripts/dev/check_coverage.py {posargs}
-
-[testenv:py35-cov]
-basepython = python3.5
-setenv = {[testenv]setenv}
-passenv = {[testenv]passenv}
-deps = {[testenv]deps}
-commands =
-    {envpython} scripts/link_pyqt.py --tox {envdir}
-    {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests}
-    {envpython} scripts/dev/check_coverage.py {posargs}
-
-[testenv:py34-cov]
-basepython = python3.4
-setenv = {[testenv]setenv}
-passenv = {[testenv]passenv}
-deps = {[testenv]deps}
-commands =
-    {envpython} scripts/link_pyqt.py --tox {envdir}
-    {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests}
-    {envpython} scripts/dev/check_coverage.py {posargs}
-
 # test envs with PyQt5 from PyPI
 
 [testenv:py35-pyqt56]
@@ -133,6 +100,34 @@ deps =
     PyQt5==5.9
 commands = {envpython} -bb -m pytest {posargs:tests}
 
+# test envs with coverage
+
+[testenv:py35-pyqt59-cov]
+basepython = python3.6
+setenv =
+    {[testenv]setenv}
+    QUTE_BDD_WEBENGINE=true
+passenv = {[testenv]passenv}
+deps =
+    {[testenv]deps}
+    PyQt5==5.9
+commands =
+    {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests}
+    {envpython} scripts/dev/check_coverage.py {posargs}
+
+[testenv:py36-pyqt59-cov]
+basepython = python3.5
+setenv =
+    {[testenv]setenv}
+    QUTE_BDD_WEBENGINE=true
+passenv = {[testenv]passenv}
+deps =
+    {[testenv]deps}
+    PyQt5==5.9
+commands =
+    {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests}
+    {envpython} scripts/dev/check_coverage.py {posargs}
+
 # other envs
 
 [testenv:mkvenv]
@@ -196,9 +191,9 @@ basepython = python3
 deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/misc/requirements/requirements-vulture.txt
+    -r{toxinidir}/misc/requirements/requirements-pyqt.txt
 setenv = PYTHONPATH={toxinidir}
 commands =
-    {envpython} scripts/link_pyqt.py --tox {envdir}
     {envpython} scripts/dev/run_vulture.py
 
 [testenv:pylint]
@@ -255,8 +250,8 @@ whitelist_externals = git
 passenv = TRAVIS TRAVIS_PULL_REQUEST
 deps =
     -r{toxinidir}/requirements.txt
+    -r{toxinidir}/misc/requirements/requirements-pyqt.txt
 commands =
-    {envpython} scripts/link_pyqt.py --tox {envdir}
     {envpython} scripts/dev/src2asciidoc.py {posargs}
     {envpython} scripts/dev/check_doc_changes.py {posargs}
     {envpython} scripts/asciidoc2html.py {posargs}
@@ -273,6 +268,8 @@ commands =
     {envbindir}/pyinstaller --noconfirm misc/qutebrowser.spec
 
 [testenv:eslint]
+# This is duplicated in travis_run.sh for Travis CI because we can't get tox in
+# the JavaScript environment easily.
 deps =
 whitelist_externals = eslint
 changedir = {toxinidir}/qutebrowser/javascript