From 35c2f95a5829ea8945425a6af0e8564f9dd68d75 Mon Sep 17 00:00:00 2001 From: Philip Scheel Date: Mon, 3 Sep 2018 23:27:09 +0200 Subject: [PATCH] Merged upstream, made requested modifications to the search engine code and added tests --- .appveyor.yml | 6 +- .github/CODEOWNERS | 2 +- .github/CONTRIBUTING.asciidoc | 5 + .travis.yml | 27 ++- README.asciidoc | 41 ++-- doc/changelog.asciidoc | 130 +++++++++++- doc/contributing.asciidoc | 13 +- doc/faq.asciidoc | 79 +++++--- doc/help/commands.asciidoc | 6 + doc/help/configuring.asciidoc | 1 + doc/help/settings.asciidoc | 52 ++++- doc/install.asciidoc | 44 ++++ doc/userscripts.asciidoc | 5 +- misc/cheatsheet.svg | 12 +- misc/qutebrowser.nsi | 3 + misc/requirements/requirements-codecov.txt | 6 +- misc/requirements/requirements-flake8.txt | 4 +- misc/requirements/requirements-pip.txt | 2 +- .../requirements/requirements-pyinstaller.txt | 6 +- .../requirements-pylint-master.txt | 6 +- .../requirements-pylint-master.txt-raw | 2 +- misc/requirements/requirements-pylint.txt | 12 +- misc/requirements/requirements-pyqt-old.txt | 4 - .../requirements-pyqt-old.txt-raw | 2 - misc/requirements/requirements-pyqt.txt | 4 +- misc/requirements/requirements-pyroma.txt | 2 +- misc/requirements/requirements-tests-git.txt | 6 +- misc/requirements/requirements-tests.txt | 22 +- misc/requirements/requirements-tests.txt-raw | 3 +- misc/requirements/requirements-tox.txt | 8 +- misc/requirements/requirements-vulture.txt | 2 +- misc/userscripts/qute-pass | 30 ++- pytest.ini | 4 + qutebrowser/__init__.py | 2 +- qutebrowser/app.py | 4 + qutebrowser/browser/adblock.py | 4 +- qutebrowser/browser/browsertab.py | 55 +++-- qutebrowser/browser/commands.py | 55 ++--- qutebrowser/browser/downloads.py | 12 +- qutebrowser/browser/downloadview.py | 2 +- qutebrowser/browser/greasemonkey.py | 5 +- qutebrowser/browser/qtnetworkdownloads.py | 11 +- qutebrowser/browser/qutescheme.py | 120 ++++++----- qutebrowser/browser/shared.py | 21 +- qutebrowser/browser/urlmarks.py | 3 +- .../browser/webengine/certificateerror.py | 7 + qutebrowser/browser/webengine/interceptor.py | 23 ++- qutebrowser/browser/webengine/spell.py | 24 ++- .../browser/webengine/webengineelem.py | 2 + .../browser/webengine/webenginequtescheme.py | 25 ++- .../browser/webengine/webenginesettings.py | 4 + qutebrowser/browser/webengine/webenginetab.py | 175 ++++++++++++---- qutebrowser/browser/webengine/webview.py | 97 ++------- qutebrowser/browser/webkit/mhtml.py | 6 +- .../browser/webkit/network/filescheme.py | 4 +- .../browser/webkit/network/networkmanager.py | 16 +- .../webkit/network/webkitqutescheme.py | 29 ++- qutebrowser/browser/webkit/webkittab.py | 8 +- qutebrowser/browser/webkit/webpage.py | 7 +- qutebrowser/completion/models/miscmodels.py | 4 +- qutebrowser/completion/models/urlmodel.py | 8 +- qutebrowser/config/configdata.py | 2 +- qutebrowser/config/configdata.yml | 63 ++++-- qutebrowser/config/configinit.py | 2 + qutebrowser/html/settings.html | 3 +- qutebrowser/javascript/.eslintrc.yaml | 2 + qutebrowser/javascript/history.js | 2 +- qutebrowser/keyinput/keyutils.py | 35 +++- qutebrowser/mainwindow/mainwindow.py | 6 +- qutebrowser/mainwindow/prompt.py | 2 +- qutebrowser/mainwindow/statusbar/url.py | 5 +- qutebrowser/mainwindow/tabbedbrowser.py | 3 +- qutebrowser/mainwindow/tabwidget.py | 8 +- qutebrowser/misc/backendproblem.py | 11 +- qutebrowser/misc/earlyinit.py | 4 +- qutebrowser/misc/editor.py | 2 - qutebrowser/misc/lineparser.py | 3 +- qutebrowser/misc/sessions.py | 2 +- qutebrowser/misc/sql.py | 8 +- qutebrowser/misc/utilcmds.py | 11 +- qutebrowser/qt.py | 28 +++ qutebrowser/utils/log.py | 31 +-- qutebrowser/utils/standarddir.py | 5 +- qutebrowser/utils/urlmatch.py | 11 + qutebrowser/utils/utils.py | 2 +- qutebrowser/utils/version.py | 32 ++- requirements.txt | 2 +- scripts/dev/build_release.py | 35 +--- scripts/dev/ci/travis_install.sh | 14 +- scripts/dictcli.py | 3 + scripts/importer.py | 2 + scripts/link_pyqt.py | 10 +- tests/conftest.py | 13 +- tests/end2end/conftest.py | 4 +- tests/end2end/data/issue4011.html | 10 + tests/end2end/data/misc/qutescheme_csrf.html | 20 ++ tests/end2end/data/userscripts/hello_if_count | 11 + tests/end2end/features/backforward.feature | 2 + tests/end2end/features/caret.feature | 4 + tests/end2end/features/downloads.feature | 1 + tests/end2end/features/editor.feature | 1 + tests/end2end/features/hints.feature | 4 +- tests/end2end/features/history.feature | 19 +- tests/end2end/features/javascript.feature | 5 +- tests/end2end/features/keyinput.feature | 2 + tests/end2end/features/misc.feature | 4 +- tests/end2end/features/open.feature | 1 + tests/end2end/features/qutescheme.feature | 64 +++++- tests/end2end/features/scroll.feature | 7 +- tests/end2end/features/search.feature | 7 +- tests/end2end/features/sessions.feature | 4 +- tests/end2end/features/spawn.feature | 11 + tests/end2end/features/tabs.feature | 1 + tests/end2end/features/test_history_bdd.py | 14 ++ tests/end2end/features/test_prompts_bdd.py | 27 ++- tests/end2end/features/urlmarks.feature | 1 + tests/end2end/features/utilcmds.feature | 9 + tests/end2end/fixtures/quteprocess.py | 13 ++ tests/end2end/fixtures/webserver_sub.py | 5 + tests/end2end/test_hints_html.py | 1 + tests/end2end/test_invocations.py | 7 +- tests/helpers/stubs.py | 6 + tests/unit/browser/test_adblock.py | 27 +++ tests/unit/browser/test_shared.py | 4 +- tests/unit/browser/webengine/test_spell.py | 80 +++++++- .../browser/webkit/network/test_filescheme.py | 6 +- tests/unit/completion/test_models.py | 189 +++++++++++++++++- tests/unit/config/test_config.py | 4 +- tests/unit/config/test_configcommands.py | 2 +- tests/unit/config/test_configinit.py | 2 + tests/unit/keyinput/key_data.py | 61 +++--- tests/unit/keyinput/test_keyutils.py | 4 +- tests/unit/misc/test_lineparser.py | 12 +- tests/unit/utils/test_qtutils.py | 27 ++- tests/unit/utils/test_standarddir.py | 7 +- tests/unit/utils/test_urlmatch.py | 23 +++ tests/unit/utils/test_utils.py | 8 +- tests/unit/utils/test_version.py | 15 +- tox.ini | 28 ++- 139 files changed, 1758 insertions(+), 652 deletions(-) delete mode 100644 misc/requirements/requirements-pyqt-old.txt delete mode 100644 misc/requirements/requirements-pyqt-old.txt-raw create mode 100644 qutebrowser/qt.py create mode 100644 tests/end2end/data/issue4011.html create mode 100644 tests/end2end/data/misc/qutescheme_csrf.html create mode 100755 tests/end2end/data/userscripts/hello_if_count diff --git a/.appveyor.yml b/.appveyor.yml index f2424fc94..92a20c0bd 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,15 +5,15 @@ cache: build: off environment: PYTHONUNBUFFERED: 1 - PYTHON: C:\Python36\python.exe + PYTHON: C:\Python36-x64\python.exe matrix: - - TESTENV: py36-pyqt510 + - TESTENV: py36-pyqt511 - TESTENV: pylint install: - '%PYTHON% -m pip install -U pip' - '%PYTHON% -m pip install -r misc\requirements\requirements-tox.txt' - - 'set PATH=%PATH%;C:\Python36' + - 'set PATH=C:\Python36-x64;%PATH' test_script: - '%PYTHON% -m tox -e %TESTENV%' diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d7da20300..1bc570a7e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,5 +1,5 @@ qutebrowser/browser/history.py @rcorre -qutebrowser/completion/* @rcorre +qutebrowser/completion/** @rcorre qutebrowser/misc/sql.py @rcorre tests/end2end/features/completion.feature @rcorre tests/end2end/features/test_completion_bdd.py @rcorre diff --git a/.github/CONTRIBUTING.asciidoc b/.github/CONTRIBUTING.asciidoc index 6449c6323..4421f071a 100644 --- a/.github/CONTRIBUTING.asciidoc +++ b/.github/CONTRIBUTING.asciidoc @@ -1,3 +1,8 @@ +IMPORTANT: I'm currently (July 2018) more busy than usual until September, +because of exams coming up. Review of non-trivial pull requests will thus be +delayed until then. If you're reading this note after mid-September, please +open an issue. + - Before you start to work on something, please leave a comment on the relevant issue (or open one). This makes sure there is no duplicate work done. diff --git a/.travis.yml b/.travis.yml index 143938aef..bb11e0c3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,18 +20,21 @@ matrix: - os: linux env: TESTENV=py36-pyqt59 - os: linux - env: TESTENV=py36-pyqt510-cov - # We need a newer Xvfb as a WORKAROUND for: - # https://bugreports.qt.io/browse/QTBUG-64928 - sudo: required + env: TESTENV=py36-pyqt510 addons: - apt: - sources: - - sourceline: "deb http://us.archive.ubuntu.com/ubuntu/ xenial main universe" - packages: - - xvfb + apt: + packages: + - xfonts-base + - os: linux + env: TESTENV=py36-pyqt511-cov + # https://github.com/travis-ci/travis-ci/issues/9069 + - os: linux + python: 3.7 + sudo: required + dist: xenial + env: TESTENV=py37-pyqt511 - os: osx - env: TESTENV=py36 OSX=sierra + env: TESTENV=py37 OSX=sierra osx_image: xcode9.2 language: generic # https://github.com/qutebrowser/qutebrowser/issues/2013 @@ -66,6 +69,10 @@ matrix: env: TESTENV=shellcheck services: docker fast_finish: true + allow_failures: + # https://github.com/qutebrowser/qutebrowser/issues/4055 + - os: linux + env: TESTENV=py36-pyqt510 cache: directories: diff --git a/README.asciidoc b/README.asciidoc index db401d49d..d587d48d3 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -59,7 +59,7 @@ Getting help You can get help in the IRC channel irc://irc.freenode.org/#qutebrowser[`#qutebrowser`] on -http://freenode.net/[Freenode] +https://freenode.net/[Freenode] (https://webchat.freenode.net/?channels=#qutebrowser[webchat]), or by writing a message to the https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[mailinglist] at @@ -96,8 +96,8 @@ Requirements The following software and libraries are required to run qutebrowser: -* http://www.python.org/[Python] 3.5 or newer (3.6 recommended) -* http://qt.io/[Qt] 5.7.1 or newer (5.10 recommended) with the following modules: +* https://www.python.org/[Python] 3.5 or newer (3.6 recommended) +* https://www.qt.io/[Qt] 5.7.1 or newer (5.11.1 recommended) with the following modules: - QtCore / qtbase - QtQuick (part of qtbase in some distributions) - QtSQL (part of qtbase in some distributions) @@ -106,14 +106,14 @@ The following software and libraries are required to run qutebrowser: - QtWebKit - only the link:https://github.com/annulen/webkit/wiki[updated fork] (5.212) is supported -* http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.7.0 or newer - (5.10 recommended) for Python 3 +* https://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.7.0 or newer + (5.11.2 recommended) for Python 3 * https://pypi.python.org/pypi/setuptools/[pkg_resources/setuptools] -* http://fdik.org/pyPEG/[pyPEG2] +* https://fdik.org/pyPEG/[pyPEG2] * http://jinja.pocoo.org/[jinja2] * http://pygments.org/[pygments] * https://github.com/yaml/pyyaml[PyYAML] -* http://www.attrs.org/[attrs] +* https://www.attrs.org/[attrs] The following libraries are optional: @@ -152,7 +152,7 @@ https://github.com/qutebrowser/qutebrowser/graphs/contributors[hundreds of contr Additionally, the following people have contributed graphics: -* Jad/link:http://yelostudio.com[yelo] (new icon) +* Jad/link:https://yelostudio.com[yelo] (new icon) * WOFall (original icon) * regines (key binding cheatsheet) @@ -170,18 +170,15 @@ Active * https://fanglingsu.github.io/vimb/[vimb] (C, GTK+ with WebKit2) * https://luakit.github.io/luakit/[luakit] (C/Lua, GTK+ with WebKit2) -* http://surf.suckless.org/[surf] (C, GTK+ with WebKit1/WebKit2) -* http://www.uzbl.org/[uzbl] (C, GTK+ with WebKit1/WebKit2) +* https://surf.suckless.org/[surf] (C, GTK+ with WebKit1/WebKit2) +* https://github.com/next-browser/next/[next] (Lisp, Emacs-like, GTK+ with WebKit) +* https://github.com/parkouss/webmacs/[webmacs] (Python, Emacs-like with QtWebEngine) * Chrome/Chromium addons: - https://github.com/1995eaton/chromium-vim[cVim], - http://vimium.github.io/[Vimium], + https://vimium.github.io/[Vimium], https://github.com/brookhong/Surfingkeys[Surfingkeys], - https://key.saka.io/[Saka Key] * Firefox addons (based on WebExtensions): https://addons.mozilla.org/en-GB/firefox/addon/vimium-ff/[Vimium-FF] (experimental), - https://key.saka.io[Saka Key], https://github.com/ueokande/vim-vixen[Vim Vixen], - https://github.com/shinglyu/QuantumVim[QuantumVim], https://github.com/cmcaine/tridactyl[Tridactyl] (working on a https://bugzilla.mozilla.org/show_bug.cgi?id=1215061[better API] for keyboard integration in Firefox). @@ -192,17 +189,23 @@ Inactive * https://bitbucket.org/portix/dwb[dwb] (C, GTK+ with WebKit1, https://bitbucket.org/portix/dwb/pull-requests/22/several-cleanups-to-increase-portability/diff[unmaintained] - main inspiration for qutebrowser) -* http://sourceforge.net/p/vimprobable/wiki/Home/[vimprobable] (C, GTK+ with +* https://sourceforge.net/p/vimprobable/wiki/Home/[vimprobable] (C, GTK+ with WebKit1) -* http://pwmt.org/projects/jumanji/[jumanji] (C, GTK+ with WebKit1) +* https://wiki.archlinux.org/index.php?title=Jumanji[jumanji] (C, GTK+ with WebKit1, +original site is gone but Arch Linux has some data) * http://conkeror.org/[conkeror] (Javascript, Emacs-like, XULRunner/Gecko) +* https://www.uzbl.org/[uzbl] (C, GTK+ with WebKit1/WebKit2) * Firefox addons (not based on WebExtensions or no recent activity): http://www.vimperator.org/[Vimperator], - http://5digits.org/pentadactyl/[Pentadactyl], + http://bug.5digits.org/pentadactyl/index[Pentadactyl], https://github.com/akhodakivskiy/VimFx[VimFx], + https://key.saka.io[Saka Key], + https://github.com/shinglyu/QuantumVim[QuantumVim], * Chrome/Chromium addons: https://chrome.google.com/webstore/detail/vichrome/gghkfhpblkcmlkmpcpgaajbbiikbhpdi?hl=en[ViChrome], https://github.com/jinzhu/vrome[Vrome] + https://key.saka.io[Saka Key], + https://github.com/1995eaton/chromium-vim[cVim], License ------- @@ -229,4 +232,4 @@ display PDF files in the browser. Windows releases come with a bundled pdf.js. pdf.js is distributed under the terms of the Apache License. You can find a copy of the license in `qutebrowser/3rdparty/pdfjs/LICENSE` (in the Windows release or after running `scripts/dev/update_3rdparty.py`), or online -http://www.apache.org/licenses/LICENSE-2.0.html[here]. +https://www.apache.org/licenses/LICENSE-2.0.html[here]. diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index b6dfa98b2..8ffdc6a35 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -15,12 +15,90 @@ breaking changes (such as renamed commands) can happen in minor releases. // `Fixed` for any bug fixes. // `Security` to invite users to upgrade in case of vulnerabilities. -v1.4.0 (unreleased) +v1.5.0 (unreleased) ------------------- Added ~~~~~ +- The qute-pass userscript now has optional OTP support. +- When `:spawn --userscript` is called with a count, that count is now + passed to userscripts as `$QUTE_COUNT`. +- New `content.mouse_lock` setting to handle HTML5 pointer locking. + +Changed +~~~~~~~ + +- The `:repeat` command now takes a count which is multiplied with the given + "times" argument. +- The default keybinding to leave passthrough mode was changed from `` + to ``, which makes pasting from the clipboard easier in + passthrough mode and is also unlikely to conflict with webpage bindings. +- The `app_id` is now set to `qutebrowser` for Wayland. +- `Command` or `Cmd` can now be used (instead of `Meta`) to map the Command key + on macOS. + +v1.4.2 (unreleased) +------------------- + +Changed +~~~~~~~ + +- The `content.xss_auditing` setting is now enabled by default, to mirror + Chromium's rather than Qt's default behavior. +- Long URLs in the statusbar are now elided at the end rather than in the + middle, to make sure the hostname is completely visible whenever possible. + +Fixed +~~~~~ + +- Crash in Qt 5.7.1 when a website uses `window.print()`. +- The workaround for Nouveau graphic drivers now works properly again. +- Crash when using `:follow-selected` with a link which is outside of the view. +- Workaround for windows not showing as urgent with some window managers + (like i3). +- Crash when opening URLs with some unicode characters (IDNA 2008). Those URLs + still won't open though, due to missing support in Qt. +- Crash when a download directory which can't be created is configured. +- Crash in the `importer.py` script when importing Chrome bookmarks from newer Chrome versions. +- The `content.webrtc_public_interfaces_only` option didn't work on Qt 5.11 previously (it now does). + Note it still does not work on Qt 5.10 (due to a Qt bug) and Qt < 5.9.2. +- Repeated escaping of entries in `qute://log` when refreshing page. +- The host blocker doesn't block 0.0.0.0 anymore. + +v1.4.1 +------ + +Security +~~~~~~~~ + +- CVE-2018-10895: Fix CSRF issue on the qute://settings page, leading to + possible arbitrary code execution. See the related GitHub issue for details: + https://github.com/qutebrowser/qutebrowser/issues/4060 + +Fixed +~~~~~ + +- Rare crash when an error occurs in downloads. +- Newlines are now stripped from the :version pastebin URL. +- There's a new `mkvenv-pypi-old` environment in `tox.ini` which installs an + older Qt, which is needed on Ubuntu 16.04. +- Worked around a Qt issue which redirects to a `chrome-error://` page when + trying to use U2F. +- The `link_pyqt.py` script now works correctly with PyQt 5.11. +- The Windows installer now uninstalls the old version before installing the + new one, fixing issues with qutebrowser not starting after installing v1.4.0 + over v1.3.3. +- The `:buffer` completion now sorts tabs with indices >= 10 correctly again. + +v1.4.0 +------ + +Added +~~~~~ + +- Support for the bundled `sip` module in PyQt 5.11 and other changes in + Qt/PyQt 5.11.x. - New `--debug-flag log-requests` to log requests to the debug log for debugging. - New `--first` flag for `:hint` (bound to `gi` for inputs) which automatically @@ -40,11 +118,14 @@ Added * Support for requesting persistent storage via `navigator.webkitPersistentStorage.requestQuota` with a new `content.persistent_storage` setting (requires Qt 5.11). + This setting also supports URL patterns. * Support for registering custom protocol handlers via `navigator.registerProtocolHandler` with a new `content.register_protocol_handler` setting (requires Qt 5.11). + This setting also supports URL patterns. * Support for WebRTC screen sharing with a new `content.desktop_capture` setting (requires Qt 5.10). + This setting also supports URL patterns. * New `content.autoplay` setting to enable/disable automatic video playback (requires Qt 5.10). * New `content.webrtc_public_interfaces_only` setting to only expose public @@ -55,8 +136,17 @@ Added Changed ~~~~~~~ +- The following settings now support URL patterns: + * `content.headers.do_not_track` + * `content.headers.custom` + * `content.headers.accept_language` + * `content.headers.user_agent` + * `content.ssl_strict` + * `content.geolocation` + * `content.notifications` + * `content.media_capture` - The Windows/macOS releases now bundle Qt 5.11.1 which is based on - Chromium 65.0.3325.151 with security fixes up to Chromium 67.0.3396.79. + Chromium 65.0.3325.151 with security fixes up to Chromium 67.0.3396.87. - New short flags for commandline arguments: `-B` and `-T` for `--basedir` and `--temp-basedir`; `-d` and `-D` for `--debug` and `--debug-flag`. - Deleting history items via `:history-clear` or `:completion-item-del` now @@ -87,26 +177,58 @@ Changed browsing. - When a prompt is opened in insert/passthrough mode, the mode is restored after closing the prompt. +- On Qt 5.10 or newer, dictionaries are now read from the qutebrowser data + directory (e.g. `~/.local/share/qutebrowser`) instead of `/usr/share/qt`. + Existing dictionaries are copied over. +- If an error while parsing `~/.netrc` occurs, the cause of the error is now + logged. +- On Qt 5.9 or newer, certificate errors now show Chromium's detailed error + page. +- Greasemonkey scripts now support a "@qute-js-world" tag to run them in a + different JavaScript context. Fixed ~~~~~ - Various subtle keyboard focus issues. +- The security fix in v1.3.3 caused URLs with ampersands + (`www.example.com?one=1&two=2`) to send the wrong arguments when clicked on + the `qute://history` page. +- Crash when opening a PDF page with PDF.js enabled (on QtWebKit), but no + PDF.js installed. +- Crash when closing a tab shortly after opening it. Removed ~~~~~~~ +- No prebuilt binaries for 32-bit Windows are supplied anymore. This is due to + Qt removing QtWebEngine support for those upstream. It might be possible to + distribute 32-bit binaries again with Qt 5.12 in December, but that will only + happen if it turns out enough people actually need 32-bit support. - `:tab-detach` which has been deprecated in v1.1.0 has been removed. - The `content.developer_extras` setting got removed. On QtWebKit, developer extras are now automatically enabled when opening the inspector. -v1.3.3 (unreleased) -------------------- +v1.3.3 +------ + +Security +~~~~~~~~ + +- An XSS vulnerability on the `qute://history` page allowed websites to inject + HTML into the page via a crafted title tag. This could allow them to steal + your browsing history. If you're currently unable to upgrade, avoid using + `:history`. A CVE request for this issue is pending, see + https://github.com/qutebrowser/qutebrowser/issues/4011[#4011] for updates. Fixed ~~~~~ - Crash in a workaround for a Qt 5.11 bug in rare circumstances. +- Workaround for a Qt bug which preserves searches between page loads. +- In v1.3.2 a dependency on the `PyQt5.QtQuickWidgets` module was accidentally + introduced. Since that module isn't packaged everywhere, it's been removed + again. v1.3.2 ------ diff --git a/doc/contributing.asciidoc b/doc/contributing.asciidoc index 07c2dfe25..031d63a22 100644 --- a/doc/contributing.asciidoc +++ b/doc/contributing.asciidoc @@ -5,6 +5,11 @@ The Compiler :data-uri: :toc: +IMPORTANT: I'm currently (July 2018) more busy than usual until September, +because of exams coming up. Review of non-trivial pull requests will thus be +delayed until then. If you're reading this note after mid-September, please +open an issue. + I `<3` footnote:[Of course, that says `<3` in HTML.] contributors! This document contains guidelines for contributing to qutebrowser, as well as @@ -88,7 +93,7 @@ git format-patch origin/master <1> Running qutebrowser ------------------- -After link:install.asciidoc#tox[installing qutebrowser via tox], you can run +After link:install.html#tox[installing qutebrowser via tox], you can run `.venv/bin/qutebrowser --debug --temp-basedir` to test your changes with debug logging enabled and without affecting existing running instances. @@ -689,8 +694,6 @@ New PyQt release ~~~~~~~~~~~~~~~~ * See above. -* Install new PyQt in Windows VM (32- and 64-bit). -* Download new installer and update PyQt installer path in `ci_install.py`. * Update `tox.ini`/`.travis.yml`/`.appveyor.yml` to test new versions. qutebrowser release @@ -712,8 +715,8 @@ qutebrowser release as closed. * Linux: Run `git checkout v1.$x.$y && ./.venv/bin/python3 scripts/dev/build_release.py --upload v1.$x.$y`. -* Windows: Run `git checkout v1.X.Y; C:\Python36-32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v1.X.Y` (replace X/Y by hand). -* macOS: Run `git checkout v1.X.Y && python3 scripts/dev/build_release.py --upload v1.X.Y` (replace X/Y by hand). +* Windows: Run `git checkout v1.X.Y; py -3.6 scripts\dev\build_release.py --asciidoc C:\Python27\python %userprofile%\bin\asciidoc-8.6.10\asciidoc.py --upload v1.X.Y` (replace X/Y by hand). +* macOS: Run `pyenv shell 3.6.6 && git checkout v1.X.Y && python3 scripts/dev/build_release.py --upload v1.X.Y` (replace X/Y by hand). * On server: - Run `python3 scripts/dev/download_release.py v1.X.Y` (replace X/Y by hand). - Run `git pull github master && sudo python3 scripts/asciidoc2html.py --website /srv/http/qutebrowser` diff --git a/doc/faq.asciidoc b/doc/faq.asciidoc index 5fd36d67b..a5d790508 100644 --- a/doc/faq.asciidoc +++ b/doc/faq.asciidoc @@ -5,10 +5,10 @@ The Compiler [qanda] What is qutebrowser based on?:: - qutebrowser uses http://www.python.org/[Python], http://qt.io/[Qt] and - http://www.riverbankcomputing.com/software/pyqt/intro[PyQt]. + qutebrowser uses https://www.python.org/[Python], https://www.qt.io/[Qt] and + https://www.riverbankcomputing.com/software/pyqt/intro[PyQt]. + -The concept of it is largely inspired by http://portix.bitbucket.org/dwb/[dwb] +The concept of it is largely inspired by https://bitbucket.org/portix/dwb/[dwb] and http://www.vimperator.org/vimperator[Vimperator]. Many actions and key bindings are similar to dwb. @@ -16,34 +16,34 @@ Why another browser?:: It might be hard to believe, but I didn't find any browser which I was happy with, so I started to write my own. Also, I needed a project to get into writing GUI applications with Python and - link:http://qt.io/[Qt]/link:http://www.riverbankcomputing.com/software/pyqt/intro[PyQt]. + link:https://www.qt.io/[Qt]/link:https://www.riverbankcomputing.com/software/pyqt/intro[PyQt]. + Read the next few questions to find out why I was unhappy with existing software. -What's wrong with link:http://portix.bitbucket.org/dwb/[dwb]/link:http://sourceforge.net/projects/vimprobable/[vimprobable]/link:https://mason-larobina.github.io/luakit/[luakit]/link:http://pwmt.org/projects/jumanji/[jumanji]/... (projects based on WebKitGTK)?:: - Most of them are based on the http://webkitgtk.org/[WebKitGTK+] - http://webkitgtk.org/reference/webkitgtk/stable/index.html[WebKit1] API, +What's wrong with link:https://bitbucket.org/portix/dwb/[dwb]/link:https://sourceforge.net/projects/vimprobable/[vimprobable]/link:https://mason-larobina.github.io/luakit/[luakit]/jumanji/... (projects based on WebKitGTK)?:: + Most of them are based on the https://webkitgtk.org/[WebKitGTK+] + https://webkitgtk.org/reference/webkitgtk/stable/index.html[WebKit1] API, which causes a lot of crashes. As the GTK API using WebKit1 is https://lists.webkit.org/pipermail/webkit-gtk/2014-March/001821.html[deprecated], these bugs are never going to be fixed. + When qutebrowser was created, the newer -http://webkitgtk.org/reference/webkit2gtk/stable/index.html[WebKit2 API] lacked +https://webkitgtk.org/reference/webkit2gtk/stable/index.html[WebKit2 API] lacked basic features like proxy support, and almost no projects have started porting to WebKit2. In the meantime, this situation has improved a bit, but there are still only a few projects which have some kind of WebKit2 support (see the https://github.com/qutebrowser/qutebrowser#similar-projects[list of alternatives]). + -qutebrowser uses http://qt.io/[Qt] and -http://wiki.qt.io/QtWebEngine[QtWebEngine] by default (and supports -http://wiki.qt.io/QtWebKit[QtWebKit] optionally). QtWebEngine is based on +qutebrowser uses https://www.qt.io/[Qt] and +https://wiki.qt.io/QtWebEngine[QtWebEngine] by default (and supports +https://wiki.qt.io/QtWebKit[QtWebKit] optionally). QtWebEngine is based on Google's https://www.chromium.org/Home[Chromium]. With an up-to-date Qt, it has much more man-power behind it than WebKitGTK+ has, and thus supports more modern web features - it's also arguably more secure. -What's wrong with https://www.mozilla.org/en-US/firefox/new/[Firefox] and link:http://5digits.org/pentadactyl/[Pentadactyl]/link:http://www.vimperator.org/vimperator[Vimperator]?:: +What's wrong with https://www.mozilla.org/en-US/firefox/new/[Firefox] and link:http://bug.5digits.org/pentadactyl/[Pentadactyl]/link:http://www.vimperator.org/vimperator[Vimperator]?:: Firefox likes to break compatibility with addons on each upgrade, gets slower and more bloated with every upgrade, and has some https://blog.mozilla.org/advancingcontent/2014/02/11/publisher-transformation-with-users-at-the-center/[horrible @@ -51,20 +51,20 @@ What's wrong with https://www.mozilla.org/en-US/firefox/new/[Firefox] and link:h + Also, developing addons for it is a nightmare. -What's wrong with http://www.chromium.org/Home[Chromium] and https://vimium.github.io/[Vimium]?:: +What's wrong with https://www.chromium.org/Home[Chromium] and https://vimium.github.io/[Vimium]?:: The Chrome plugin API doesn't seem to allow much freedom for plugin writers, which results in Vimium not really having all the features you'd expect from a proper minimal, vim-like browser. Why Python?:: I enjoy writing Python since 2011, which made it one of the possible - choices. I wanted to use http://qt.io/[Qt] because of - http://wiki.qt.io/QtWebKit[QtWebKit] so I didn't have - http://wiki.qt.io/Category:LanguageBindings[many other choices]. I don't + choices. I wanted to use https://www.qt.io/[Qt] because of + https://wiki.qt.io/QtWebKit[QtWebKit] so I didn't have + https://wiki.qt.io/Category:LanguageBindings[many other choices]. I don't like C++ and can't write it very well, so that wasn't an alternative. But isn't Python too slow for a browser?:: - http://www.infoworld.com/d/application-development/van-rossum-python-not-too-slow-188715[No.] + https://www.infoworld.com/d/application-development/van-rossum-python-not-too-slow-188715[No.] I believe efficiency while coding is a lot more important than efficiency while running. Also, most of the heavy lifting of qutebrowser is done by Qt and WebKit in C++, with the @@ -74,7 +74,7 @@ Is qutebrowser secure?:: Most security issues are in the backend (which handles networking, rendering, JavaScript, etc.) and not qutebrowser itself. + -qutebrowser uses http://wiki.qt.io/QtWebEngine[QtWebEngine] by default. +qutebrowser uses https://wiki.qt.io/QtWebEngine[QtWebEngine] by default. QtWebEngine is based on Google's https://www.chromium.org/Home[Chromium]. While Qt only updates to a new Chromium release on every minor Qt release (all ~6 months), every patch release backports security fixes from newer Chromium @@ -84,26 +84,41 @@ do anything. Chromium's process isolation and https://chromium.googlesource.com/chromium/src/+/master/docs/design/sandbox.md[sandboxing] features are also enabled as a second line of defense. + -http://wiki.qt.io/QtWebKit[QtWebKit] is also supported as an alternative +https://wiki.qt.io/QtWebKit[QtWebKit] is also supported as an alternative backend, but hasn't seen new releases https://github.com/annulen/webkit/releases[in a while]. It also doesn't have any -process isolation or sandboxing. +process isolation or sandboxing. See +https://github.com/qutebrowser/qutebrowser/issues/4039[#4039] for more details. + -Security issues in qutebrowser's code happen very rarely (as per March 2018, -there has been one security issue caused by qutebrowser in over four years) and -are fixed timely. To report security bugs, please contact me directly at -mail@qutebrowser.org, GPG ID +Security issues in qutebrowser's code happen very rarely (as per July 2018, +there have been three security issues caused by qutebrowser in over 4.5 years). +Those were handled appropriately +(http://seclists.org/oss-sec/2018/q3/29[example]) and fixed timely. To report +security bugs, please contact me directly at mail@qutebrowser.org, GPG ID https://www.the-compiler.org/pubkey.asc[0x916eb0c8fd55a072]. Is there an adblocker?:: There is a host-based adblocker which takes /etc/hosts-like lists. A "real" adblocker has a - http://www.reddit.com/r/programming/comments/25j41u/adblock_pluss_effect_on_firefoxs_memory_usage/chhpomw[big + https://www.reddit.com/r/programming/comments/25j41u/adblock_pluss_effect_on_firefoxs_memory_usage/chhpomw[big impact] on browsing speed and https://blog.mozilla.org/nnethercote/2014/05/14/adblock-pluss-effect-on-firefoxs-memory-usage/[RAM usage], so implementing support for AdBlockPlus-like lists is currently not a priority. +How can I get No-Script-like behavior?:: + To disable JavaScript by default: ++ +---- +:set content.javascript.enabled false +---- ++ +The basic command for enabling JavaScript for the current host is `tsh`. +This will allow JavaScript execution for the current session. +Use `S` instead of `s` to make the exception permanent. +With `H` instead of `h`, subdomains are included. +With `u` instead of `h`, only the current URL is whitelisted (not the whole host). + How do I play Youtube videos with mpv?:: You can easily add a key binding to play youtube videos inside a real video player - optionally even with hinting for links: @@ -145,11 +160,11 @@ What is the difference between bookmarks and quickmarks?:: Bookmarks will always use the title of the website as their name, but with quickmarks you can set your own title. + -For example, if you bookmark multiple food recipe websites and use `:open`, +For example, if you bookmark multiple food recipe websites and use `:open`, you have to type the title or address of the website. + When using quickmark, you can give them all names, like -`foodrecipes1`, `foodrecipes2` and so on. When you type +`foodrecipes1`, `foodrecipes2` and so on. When you type `:open foodrecipes`, you will see a list of all the food recipe sites, without having to remember the exact website title or address. @@ -243,7 +258,7 @@ Note that there are some missing features which you may run into: See https://wiki.greasespot.net/Metadata_Block[the wiki] for more info. . Any greasemonkey API function to do with adding UI elements is not currently supported. That means context menu extentensions and background pages. - + == Troubleshooting Unable to view flash content.:: @@ -251,7 +266,7 @@ Unable to view flash content.:: to use the flash plugin. Using the command `:set content.plugins true` in qutebrowser will enable plugins. Packages for flash should be provided for your platform or it can be obtained from - http://get.adobe.com/flashplayer/[Adobe]. + https://get.adobe.com/flashplayer/[Adobe]. Experiencing freezing on sites like duckduckgo and youtube.:: This issue could be caused by stale plugin files installed by `mozplugger` @@ -264,12 +279,12 @@ When using QtWebEngine, qutebrowser reports "Render Process Crashed" and the con As stated in https://gcc.gnu.org/gcc-6/changes.html[GCC's Website] GCC 6 has introduced some optimizations that could break non-conforming codebases, like QtWebEngine. + As a workaround, you can disable the nullpointer check optimization by adding the -fno-delete-null-pointer-checks flag while compiling. + On gentoo, you just need to add it into your make.conf, like this: + - + CFLAGS="... -fno-delete-null-pointer-checks" CXXFLAGS="... -fno-delete-null-pointer-checks" + And then re-emerging qtwebengine with: + - + emerge -1 qtwebengine Unable to view DRM content (Netflix, Spotify, etc.).:: @@ -300,5 +315,5 @@ My issue is not listed.:: https://github.com/qutebrowser/qutebrowser/issues[the issue tracker] or using the `:report` command. If you are reporting a segfault, make sure you read the - link:stacktrace.asciidoc[guide] on how to report them with all needed + link:doc/stacktrace.html[guide] on how to report them with all needed information. diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 3d60cce64..04cf5f318 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -918,6 +918,9 @@ Repeat a given command. * +'times'+: How many times to repeat. * +'command'+: The command to run, with optional args. +==== count +Multiplies with 'times' when given. + ==== note * This command does not split arguments after the last argument and handles quotes literally. * With this command, +;;+ is interpreted literally instead of splitting off a second command. @@ -1197,6 +1200,9 @@ Spawn a command in a shell. * +*-o*+, +*--output*+: Whether the output should be shown in a new tab. * +*-d*+, +*--detach*+: Whether the command should be detached from qutebrowser. +==== count +Given to userscripts as $QUTE_COUNT. + ==== note * This command does not split arguments after the last argument and handles quotes literally. diff --git a/doc/help/configuring.asciidoc b/doc/help/configuring.asciidoc index 4734e6b54..362f1682e 100644 --- a/doc/help/configuring.asciidoc +++ b/doc/help/configuring.asciidoc @@ -395,6 +395,7 @@ Pre-built colorschemes - A collection of https://github.com/chriskempson/base16[base16] color-schemes can be found in https://github.com/theova/base16-qutebrowser[base16-qutebrowser] and used with https://github.com/AuditeMarlow/base16-manager[base16-manager]. - Two implementations of the https://github.com/arcticicestudio/nord[Nord] colorscheme for qutebrowser exist: https://github.com/Linuus/nord-qutebrowser[Linuus], https://github.com/KnownAsDon/QuteBrowser-Nord-Theme[KnownAsDon] +- https://github.com/evannagle/qutebrowser-dracula-theme[Dracula] Avoiding flake8 errors ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 8bac566da..ef06ce7e6 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -143,6 +143,7 @@ |<>|Allow locally loaded documents to access remote URLs. |<>|Enable support for HTML 5 local storage and Web SQL. |<>|Allow websites to record audio/video. +|<>|Allow websites to lock your mouse pointer. |<>|Netrc-file for HTTP authentication. |<>|Allow websites to show notifications. |<>|Allow pdf.js to view PDF files in the browser. @@ -563,6 +564,7 @@ Default: * +pass:[g0]+: +pass:[tab-focus 1]+ * +pass:[gB]+: +pass:[set-cmd-text -s :bookmark-load -t]+ * +pass:[gC]+: +pass:[tab-clone]+ +* +pass:[gD]+: +pass:[tab-give]+ * +pass:[gO]+: +pass:[set-cmd-text :open -t -r {url:pretty}]+ * +pass:[gU]+: +pass:[navigate up -t]+ * +pass:[g^]+: +pass:[tab-focus 1]+ @@ -634,7 +636,7 @@ Default: * +pass:[}}]+: +pass:[navigate next -t]+ - +pass:[passthrough]+: -* +pass:[<Ctrl-V>]+: +pass:[leave-mode]+ +* +pass:[<Shift-Escape>]+: +pass:[leave-mode]+ - +pass:[prompt]+: * +pass:[<Alt-B>]+: +pass:[rl-backward-word]+ @@ -1571,6 +1573,8 @@ Default: +pass:[iso-8859-1]+ Allow websites to share screen content. On Qt < 5.10, a dialog box is always displayed, even if this is set to "true". +This setting supports URL patterns. + Type: <> Valid values: @@ -1610,6 +1614,8 @@ This setting is only available with the QtWebKit backend. === content.geolocation Allow websites to request geolocations. +This setting supports URL patterns. + Type: <> Valid values: @@ -1623,6 +1629,9 @@ Default: +pass:[ask]+ [[content.headers.accept_language]] === content.headers.accept_language Value to send in the `Accept-Language` header. +Note that the value read from JavaScript is always the global value. + +This setting supports URL patterns. Type: <> @@ -1632,6 +1641,8 @@ Default: +pass:[en-US,en]+ === content.headers.custom Custom headers for qutebrowser HTTP requests. +This setting supports URL patterns. + Type: <> Default: empty @@ -1641,6 +1652,8 @@ Default: empty Value to send in the `DNT` header. When this is set to true, qutebrowser asks websites to not track your identity. If set to null, the DNT header is not sent at all. +This setting supports URL patterns. + Type: <> Default: +pass:[true]+ @@ -1665,6 +1678,9 @@ This setting is only available with the QtWebKit backend. [[content.headers.user_agent]] === content.headers.user_agent User agent to send. Unset to send the default. +Note that the value read from JavaScript is always the global value. + +This setting supports URL patterns. Type: <> @@ -1844,6 +1860,8 @@ Default: +pass:[true]+ === content.media_capture Allow websites to record audio/video. +This setting supports URL patterns. + Type: <> Valid values: @@ -1856,6 +1874,26 @@ Default: +pass:[ask]+ This setting is only available with the QtWebEngine backend. +[[content.mouse_lock]] +=== content.mouse_lock +Allow websites to lock your mouse pointer. + +This setting supports URL patterns. + +Type: <> + +Valid values: + + * +true+ + * +false+ + * +ask+ + +Default: +pass:[ask]+ + +On QtWebEngine, this setting requires Qt 5.8 or newer. + +On QtWebKit, this setting is unavailable. + [[content.netrc_file]] === content.netrc_file Netrc-file for HTTP authentication. @@ -1869,6 +1907,8 @@ Default: empty === content.notifications Allow websites to show notifications. +This setting supports URL patterns. + Type: <> Valid values: @@ -1896,6 +1936,8 @@ This setting is only available with the QtWebKit backend. === content.persistent_storage Allow websites to request persistent storage quota via `navigator.webkitPersistentStorage.requestQuota`. +This setting supports URL patterns. + Type: <> Valid values: @@ -1968,6 +2010,8 @@ This setting is only available with the QtWebKit backend. === content.register_protocol_handler Allow websites to register protocol handlers via `navigator.registerProtocolHandler`. +This setting supports URL patterns. + Type: <> Valid values: @@ -1986,6 +2030,8 @@ On QtWebKit, this setting is unavailable. === content.ssl_strict Validate SSL handshakes. +This setting supports URL patterns. + Type: <> Valid values: @@ -2038,13 +2084,13 @@ Default: +pass:[false]+ [[content.xss_auditing]] === content.xss_auditing Monitor load requests for cross-site scripting attempts. -Suspicious scripts will be blocked and reported in the inspector's JavaScript console. Enabling this feature might have an impact on performance. +Suspicious scripts will be blocked and reported in the inspector's JavaScript console. This setting supports URL patterns. Type: <> -Default: +pass:[false]+ +Default: +pass:[true]+ [[downloads.location.directory]] === downloads.location.directory diff --git a/doc/install.asciidoc b/doc/install.asciidoc index ddf1cd120..6e3eb7b25 100644 --- a/doc/install.asciidoc +++ b/doc/install.asciidoc @@ -222,6 +222,38 @@ There are prebuilt RPMs available at https://software.opensuse.org/download.html To use the QtWebEngine backend, install `libqt5-qtwebengine`. +On Slackware +------------ + +qutebrowser is available in the 3rd party repository at http://slackbuilds.org[slackbuilds.org] + +An easy way to install it is with sbopkg (frontend for slackbuilds.org) available at http://sbopkg.org[sbopkg.org] + +sbopkg can be run with a dialog screen interface, or via command line options. + +After installing the latest sbopkg package, choose your release version, and sync the repo. + +---- +sbopkg -V 14.2 +sbopkg -r +---- + +The pyPEG2 and MarkupSafe dependencies both need building for python3. You can either set PYTHON3=yes in the shell or set those as options in the dialog menu for each. + +Generate a queue file for qutebrowser and dependencies: + +---- +sqg -p qutebrowser +---- + +Then load the queue in the dialog queue menu or via: + +---- +PYTHON3=yes sbopkg -i qutebrowser +---- + +If you use the dialog screen you can deselect any already-installed packages that you don't need/want to rebuild before starting the build process. + On OpenBSD ---------- @@ -392,6 +424,10 @@ https://docs.python.org/3/library/venv.html[virtual environment]: $ tox -e mkvenv-pypi ---- +If your system comes with Python 3.5.3 or older (such as Ubuntu 16.04 LTS), use +`tox -e mkvenv-pypi-old` instead. This installs an older Qt version (5.10) due +to bugs in newer versions. + This installs all needed Python dependencies in a `.venv` subfolder. This comes with an up-to-date Qt/PyQt including QtWebEngine, but has a few @@ -407,6 +443,14 @@ caveats: (`export LD_LIBRARY_PATH=/usr/lib/openssl-1.0` on Archlinux) before starting qutebrowser if you want SSL to work in certain downloads (e.g. for `:adblock-update` or `:download`). + * On Ubuntu (tested on 18.04), you will need to install the `libssl1.0.0` + package (`apt install libssl1.0.0`). Then, in the qutebrowser git + repository, create a directory named `libssl` (`mkdir libssl`), and link + `libcrypto.so.1.0.0` and `libssl.so.1.0.0` into it without the versioning + part in their names (`ln -s /usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0 + libssl/libcrypto.so` and `ln -s /usr/lib/x86_64-linux-gnu/libssl.so.1.0.0 + libssl/libssl.so`). Now you can start qutebrowser issuing `export + LD_LIBRARY_PATH=$(pwd)/libssl` beforehand. - It comes with a QtWebEngine compiled without proprietary codec support (such as h.264). diff --git a/doc/userscripts.asciidoc b/doc/userscripts.asciidoc index c2f35b026..2dc34402d 100644 --- a/doc/userscripts.asciidoc +++ b/doc/userscripts.asciidoc @@ -19,7 +19,7 @@ Also note userscripts need to have the executable bit set (`chmod +x`) for qutebrowser to run them. To call a userscript, it needs to be stored in your data directory under -`userscripts` (for example: `~/.local/share/qutebrowser/userscripts/myscript`), +`userscripts` (for example: `~/.local/share/qutebrowser/userscripts/myscript`), or just use an absolute path. NOTE: On Windows, only userscripts with `com`, `bat`, or `exe` extensions will be launched. @@ -45,12 +45,13 @@ In `command` mode: - `QUTE_URL`: The current URL. - `QUTE_TITLE`: The title of the current page. - `QUTE_SELECTED_TEXT`: The text currently selected on the page. +- `QUTE_COUNT`: The `count` from the spawn command running the userscript. In `hints` mode: - `QUTE_URL`: The URL selected via hints. - `QUTE_SELECTED_TEXT`: The plain text of the element selected via hints. -- `QUTE_SELECTED_HTML` The HTML of the element selected via hints. +- `QUTE_SELECTED_HTML`: The HTML of the element selected via hints. Sending commands ---------------- diff --git a/misc/cheatsheet.svg b/misc/cheatsheet.svg index bb87142a4..e792489d4 100644 --- a/misc/cheatsheet.svg +++ b/misc/cheatsheet.svg @@ -13,7 +13,7 @@ height="682.66669" id="svg2" sodipodi:version="0.32" - inkscape:version="0.92.2 5c3e80d, 2017-08-06" + inkscape:version="0.92.2 2405546, 2018-03-11" version="1.0" sodipodi:docname="cheatsheet.svg" inkscape:output_extension="org.inkscape.output.svg.inkscape" @@ -33,7 +33,7 @@ inkscape:pageopacity="0.0" inkscape:pageshadow="2" inkscape:zoom="1.7536248" - inkscape:cx="430.72917" + inkscape:cx="613.20834" inkscape:cy="268.64059" inkscape:document-units="px" inkscape:current-layer="layer1" @@ -3085,7 +3085,9 @@ style="font-size:10.66666698px;line-height:1.25;font-family:sans-serif;fill:#000000;stroke-width:1.06666672" id="flowPara4056"> (to index/left/right)gC - clone tab  gC - clone tabgD - detach tab gf - view page sourcesf - save configss - set settingss - set setting (sl: temp)sl - set temp. settingsk - bind key[PASSWORD], which is compatible with almost all login forms.""" +[USERNAME][PASSWORD], which is compatible with almost all login forms. -EPILOG = """Dependencies: tldextract (Python 3 module), pass. +Suggested bindings similar to Uzbl's `formfiller` script: + + config.bind('', 'spawn --userscript qute-pass') + config.bind('', 'spawn --userscript qute-pass --username-only') + config.bind('

', 'spawn --userscript qute-pass --password-only') + config.bind('', 'spawn --userscript qute-pass --otp-only') +""" + +EPILOG = """Dependencies: tldextract (Python 3 module), pass, pass-otp (optional). For issues and feedback please use: https://github.com/cryzed/qutebrowser-userscripts. WARNING: The login details are viewable as plaintext in qutebrowser's debug log (qute://log) and might be shared if @@ -66,6 +74,7 @@ argument_parser.add_argument('--merge-candidates', '-m', action='store_true', group = argument_parser.add_mutually_exclusive_group() group.add_argument('--username-only', '-e', action='store_true', help='Only insert username') group.add_argument('--password-only', '-w', action='store_true', help='Only insert password') +group.add_argument('--otp-only', '-o', action='store_true', help='Only insert OTP code') stderr = functools.partial(print, file=sys.stderr) @@ -87,7 +96,7 @@ def qute_command(command): def find_pass_candidates(domain, password_store_path): candidates = [] - for path, directories, file_names in os.walk(password_store_path): + for path, directories, file_names in os.walk(password_store_path, followlinks=True): if directories or domain not in path.split(os.path.sep): continue @@ -98,11 +107,19 @@ def find_pass_candidates(domain, password_store_path): return candidates -def pass_(path, encoding): - process = subprocess.run(['pass', path], stdout=subprocess.PIPE) +def _run_pass(command, encoding): + process = subprocess.run(command, stdout=subprocess.PIPE) return process.stdout.decode(encoding).strip() +def pass_(path, encoding): + return _run_pass(['pass', path], encoding) + + +def pass_otp(path, encoding): + return _run_pass(['pass', 'otp', path], encoding) + + def dmenu(items, invocation, encoding): command = shlex.split(invocation) process = subprocess.run(command, input='\n'.join(items).encode(encoding), stdout=subprocess.PIPE) @@ -169,6 +186,9 @@ def main(arguments): fake_key_raw(username) elif arguments.password_only: fake_key_raw(password) + elif arguments.otp_only: + otp = pass_otp(selection, arguments.io_encoding) + fake_key_raw(otp) else: # Enter username and password using fake-key and (which seems to work almost universally), then switch # back into insert-mode, so the form can be directly submitted by hitting enter afterwards diff --git a/pytest.ini b/pytest.ini index c897f0be7..1b5016321 100644 --- a/pytest.ini +++ b/pytest.ini @@ -63,4 +63,8 @@ qt_log_ignore = ^inotify_add_watch\(".*"\) failed: "No space left on device" ^QSettings::value: Empty key passed ^Icon theme ".*" not found + ^Error receiving trust for a CA certificate xfail_strict = true +filterwarnings = + # This happens in many qutebrowser dependencies... + ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working:DeprecationWarning diff --git a/qutebrowser/__init__.py b/qutebrowser/__init__.py index 770f014ea..fab8a5fae 100644 --- a/qutebrowser/__init__.py +++ b/qutebrowser/__init__.py @@ -26,7 +26,7 @@ __copyright__ = "Copyright 2014-2018 Florian Bruhin (The Compiler)" __license__ = "GPL" __maintainer__ = __author__ __email__ = "mail@qutebrowser.org" -__version_info__ = (1, 3, 2) +__version_info__ = (1, 4, 1) __version__ = '.'.join(str(e) for e in __version_info__) __description__ = "A keyboard-driven, vim-like browser based on PyQt5." diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 3cce2e85e..ed2a85eed 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -104,6 +104,7 @@ def run(args): qApp = Application(args) qApp.setOrganizationName("qutebrowser") qApp.setApplicationName("qutebrowser") + qApp.setDesktopFileName("qutebrowser") qApp.setApplicationVersion(qutebrowser.__version__) qApp.lastWindowClosed.connect(quitter.on_last_window_closed) @@ -129,6 +130,9 @@ def run(args): sys.exit(usertypes.Exit.err_ipc) if server is None: + if args.backend is not None: + log.init.warning( + "Backend from the running instance will be used") sys.exit(usertypes.Exit.ok) else: server.got_args.connect(lambda args, target_arg, cwd: diff --git a/qutebrowser/browser/adblock.py b/qutebrowser/browser/adblock.py index f42d1a1db..affd80eaf 100644 --- a/qutebrowser/browser/adblock.py +++ b/qutebrowser/browser/adblock.py @@ -234,7 +234,9 @@ class HostBlocker: hosts = parts[1:] for host in hosts: - if '.' in host and not host.endswith('.localdomain'): + if ('.' in host and + not host.endswith('.localdomain') and + host != '0.0.0.0'): self._blocked_hosts.add(host) return True diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index 7e78b2621..087834e53 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -22,11 +22,11 @@ import enum import itertools -import sip import attr from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QSizeF, Qt from PyQt5.QtGui import QIcon -from PyQt5.QtWidgets import QWidget, QApplication +from PyQt5.QtWidgets import QWidget, QApplication, QDialog +from PyQt5.QtPrintSupport import QPrintDialog import pygments import pygments.lexers @@ -38,6 +38,7 @@ from qutebrowser.utils import (utils, objreg, usertypes, log, qtutils, urlutils, message) from qutebrowser.misc import miscwidgets, objects from qutebrowser.browser import mouse, hints +from qutebrowser.qt import sip tab_id_gen = itertools.count(0) @@ -187,8 +188,9 @@ class AbstractPrinting: """Attribute of AbstractTab for printing the page.""" - def __init__(self): + def __init__(self, tab): self._widget = None + self._tab = tab def check_pdf_support(self): raise NotImplementedError @@ -212,6 +214,29 @@ class AbstractPrinting: """ raise NotImplementedError + def show_dialog(self): + """Print with a QPrintDialog.""" + self.check_printer_support() + + def print_callback(ok): + """Called when printing finished.""" + if not ok: + message.error("Printing failed!") + diag.deleteLater() + + def do_print(): + """Called when the dialog was closed.""" + self.to_printer(diag.printer(), print_callback) + + diag = QPrintDialog(self._tab) + if utils.is_mac: + # For some reason we get a segfault when using open() on macOS + ret = diag.exec_() + if ret == QDialog.Accepted: + do_print() + else: + diag.open(do_print) + class AbstractSearch(QObject): @@ -829,12 +854,20 @@ class AbstractTab(QWidget): navigation.navigation_type, navigation.is_main_frame)) - if (navigation.navigation_type == navigation.Type.link_clicked and - not navigation.url.isValid()): - msg = urlutils.get_errstring(navigation.url, - "Invalid link clicked") - message.error(msg) - self.data.open_target = usertypes.ClickTarget.normal + if not navigation.url.isValid(): + # Also a WORKAROUND for missing IDNA 2008 support in QUrl, see + # https://bugreports.qt.io/browse/QTBUG-60364 + + if navigation.navigation_type == navigation.Type.link_clicked: + msg = urlutils.get_errstring(navigation.url, + "Invalid link clicked") + message.error(msg) + self.data.open_target = usertypes.ClickTarget.normal + + log.webview.debug("Ignoring invalid URL {} in " + "acceptNavigationRequest: {}".format( + navigation.url.toDisplayString(), + navigation.url.errorString())) navigation.accepted = False def handle_auto_insert_mode(self, ok): @@ -893,10 +926,6 @@ class AbstractTab(QWidget): self._progress = perc self.load_progress.emit(perc) - @pyqtSlot() - def _on_ssl_errors(self): - self._has_ssl_errors = True - def url(self, requested=False): raise NotImplementedError diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index b73d86a87..d5e1797ac 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -25,9 +25,9 @@ import shlex import functools import typing -from PyQt5.QtWidgets import QApplication, QTabBar, QDialog +from PyQt5.QtWidgets import QApplication, QTabBar from PyQt5.QtCore import pyqtSlot, Qt, QUrl, QEvent, QUrlQuery -from PyQt5.QtPrintSupport import QPrintDialog, QPrintPreviewDialog +from PyQt5.QtPrintSupport import QPrintPreviewDialog from qutebrowser.commands import userscripts, cmdexc, cmdutils, runners from qutebrowser.config import config, configdata @@ -415,27 +415,6 @@ class CommandDispatcher: tab.printing.to_pdf(filename) log.misc.debug("Print to file: {}".format(filename)) - def _print(self, tab): - """Print with a QPrintDialog.""" - def print_callback(ok): - """Called when printing finished.""" - if not ok: - message.error("Printing failed!") - diag.deleteLater() - - def do_print(): - """Called when the dialog was closed.""" - tab.printing.to_printer(diag.printer(), print_callback) - - diag = QPrintDialog(tab) - if utils.is_mac: - # For some reason we get a segfault when using open() on macOS - ret = diag.exec_() - if ret == QDialog.Accepted: - do_print() - else: - diag.open(do_print) - @cmdutils.register(instance='command-dispatcher', name='print', scope='window') @cmdutils.argument('count', count=True) @@ -453,22 +432,15 @@ class CommandDispatcher: return try: - if pdf: - tab.printing.check_pdf_support() - else: - tab.printing.check_printer_support() if preview: - tab.printing.check_preview_support() + self._print_preview(tab) + elif pdf: + self._print_pdf(tab, pdf) + else: + tab.printing.show_dialog() except browsertab.WebTabError as e: raise cmdexc.CommandError(e) - if preview: - self._print_preview(tab) - elif pdf: - self._print_pdf(tab, pdf) - else: - self._print(tab) - @cmdutils.register(instance='command-dispatcher', scope='window') def tab_clone(self, bg=False, window=False): """Duplicate the current tab. @@ -1201,8 +1173,9 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0, no_replace_variables=True) + @cmdutils.argument('count', count=True) def spawn(self, cmdline, userscript=False, verbose=False, - output=False, detach=False): + output=False, detach=False, count=None): """Spawn a command in a shell. Args: @@ -1216,6 +1189,7 @@ class CommandDispatcher: output: Whether the output should be shown in a new tab. detach: Whether the command should be detached from qutebrowser. cmdline: The commandline to execute. + count: Given to userscripts as $QUTE_COUNT. """ cmdutils.check_exclusive((userscript, detach), 'ud') try: @@ -1239,7 +1213,7 @@ class CommandDispatcher: if userscript: def _selection_callback(s): try: - runner = self._run_userscript(s, cmd, args, verbose) + runner = self._run_userscript(s, cmd, args, verbose, count) runner.finished.connect(_on_proc_finished) except cmdexc.CommandError as e: message.error(str(e)) @@ -1266,19 +1240,23 @@ class CommandDispatcher: """Open main startpage in current tab.""" self.openurl(config.val.url.start_pages[0]) - def _run_userscript(self, selection, cmd, args, verbose): + def _run_userscript(self, selection, cmd, args, verbose, count): """Run a userscript given as argument. Args: cmd: The userscript to run. args: Arguments to pass to the userscript. verbose: Show notifications when the command started/exited. + count: Exposed to the userscript. """ env = { 'QUTE_MODE': 'command', 'QUTE_SELECTED_TEXT': selection, } + if count is not None: + env['QUTE_COUNT'] = str(count) + idx = self._current_index() if idx != -1: env['QUTE_TITLE'] = self._tabbed_browser.widget.page_title(idx) @@ -1455,6 +1433,7 @@ class CommandDispatcher: if tab.data.inspector is None: tab.data.inspector = inspector.create() tab.data.inspector.inspect(page) + tab.data.inspector.show() else: tab.data.inspector.toggle(page) except inspector.WebInspectorError as e: diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py index dd112e00a..854fd1ca5 100644 --- a/qutebrowser/browser/downloads.py +++ b/qutebrowser/browser/downloads.py @@ -29,7 +29,6 @@ import pathlib import tempfile import enum -import sip from PyQt5.QtCore import (pyqtSlot, pyqtSignal, Qt, QObject, QModelIndex, QTimer, QAbstractListModel, QUrl) @@ -37,6 +36,7 @@ from qutebrowser.commands import cmdexc, cmdutils from qutebrowser.config import config from qutebrowser.utils import (usertypes, standarddir, utils, message, log, qtutils) +from qutebrowser.qt import sip ModelRole = enum.IntEnum('ModelRole', ['item'], start=Qt.UserRole) @@ -80,9 +80,9 @@ def download_dir(): ddir = directory try: - os.makedirs(ddir) - except FileExistsError: - pass + os.makedirs(ddir, exist_ok=True) + except OSError as e: + message.error("Failed to create download directory: {}".format(e)) return ddir @@ -692,9 +692,7 @@ class AbstractDownloadItem(QObject): global last_used_directory try: - os.makedirs(os.path.dirname(self._filename)) - except FileExistsError: - pass + os.makedirs(os.path.dirname(self._filename), exist_ok=True) except OSError as e: self._die(e.strerror) diff --git a/qutebrowser/browser/downloadview.py b/qutebrowser/browser/downloadview.py index 80da117d2..e90e37509 100644 --- a/qutebrowser/browser/downloadview.py +++ b/qutebrowser/browser/downloadview.py @@ -21,13 +21,13 @@ import functools -import sip from PyQt5.QtCore import pyqtSlot, QSize, Qt, QTimer from PyQt5.QtWidgets import QListView, QSizePolicy, QMenu, QStyleFactory from qutebrowser.browser import downloads from qutebrowser.config import config from qutebrowser.utils import qtutils, utils, objreg +from qutebrowser.qt import sip def update_geometry(obj): diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index db8246bab..7e5ff7f5b 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -58,6 +58,7 @@ class GreasemonkeyScript: self.run_at = None self.script_meta = None self.runs_on_sub_frames = True + self.jsworld = "main" for name, value in properties: if name == 'name': self.name = value @@ -77,6 +78,8 @@ class GreasemonkeyScript: self.runs_on_sub_frames = False elif name == 'require': self.requires.append(value) + elif name == 'qute-js-world': + self.jsworld = value HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n' PROPS_REGEX = r'// @(?P[^\s]+)\s*(?P.*)' @@ -142,7 +145,7 @@ class GreasemonkeyScript: @attr.s -class MatchingScripts(object): +class MatchingScripts: """All userscripts registered to run on a particular url.""" diff --git a/qutebrowser/browser/qtnetworkdownloads.py b/qutebrowser/browser/qtnetworkdownloads.py index 4e992b172..bdd2b280a 100644 --- a/qutebrowser/browser/qtnetworkdownloads.py +++ b/qutebrowser/browser/qtnetworkdownloads.py @@ -29,7 +29,7 @@ from PyQt5.QtCore import pyqtSlot, pyqtSignal, QTimer from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply from qutebrowser.config import config -from qutebrowser.utils import message, usertypes, log, urlutils, utils +from qutebrowser.utils import message, usertypes, log, urlutils, utils, debug from qutebrowser.browser import downloads from qutebrowser.browser.webkit import http from qutebrowser.browser.webkit.network import networkmanager @@ -307,7 +307,14 @@ class DownloadItem(downloads.AbstractDownloadItem): """Handle QNetworkReply errors.""" if code == QNetworkReply.OperationCanceledError: return - self._die(self._reply.errorString()) + + if self._reply is None: + error = "Unknown error: {}".format( + debug.qenum_key(QNetworkReply, code)) + else: + error = self._reply.errorString() + + self._die(error) @pyqtSlot() def _on_read_timer_timeout(self): diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 6fc2a8429..ee33d7897 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -24,6 +24,7 @@ Module attributes: _HANDLERS: The handlers registered via decorators. """ +import html import json import os import time @@ -31,20 +32,29 @@ import textwrap import mimetypes import urllib import collections +import base64 + +try: + import secrets +except ImportError: + # New in Python 3.6 + secrets = None import pkg_resources -import sip from PyQt5.QtCore import QUrlQuery, QUrl +from PyQt5.QtNetwork import QNetworkReply import qutebrowser from qutebrowser.config import config, configdata, configexc, configdiff from qutebrowser.utils import (version, utils, jinja, log, message, docutils, objreg, urlutils) from qutebrowser.misc import objects +from qutebrowser.qt import sip pyeval_output = ":pyeval was never called" spawn_output = ":spawn was never called" +csrf_token = None _HANDLERS = {} @@ -123,12 +133,12 @@ class add_handler: # noqa: N801,N806 pylint: disable=invalid-name def wrong_backend_handler(self, url): """Show an error page about using the invalid backend.""" - html = jinja.render('error.html', - title="Error while opening qute://url", - url=url.toDisplayString(), - error='{} is not available with this ' - 'backend'.format(url.toDisplayString())) - return 'text/html', html + src = jinja.render('error.html', + title="Error while opening qute://url", + url=url.toDisplayString(), + error='{} is not available with this ' + 'backend'.format(url.toDisplayString())) + return 'text/html', src def data_for_url(url): @@ -177,8 +187,6 @@ def data_for_url(url): except OSError as e: # FIXME:qtwebengine how to handle this? raise QuteSchemeOSError(e) - except QuteSchemeError: - raise assert mimetype is not None, url if mimetype == 'text/html' and isinstance(data, str): @@ -196,11 +204,11 @@ def qute_bookmarks(_url): quickmarks = sorted(objreg.get('quickmark-manager').marks.items(), key=lambda x: x[0]) # Sort by name - html = jinja.render('bookmarks.html', - title='Bookmarks', - bookmarks=bookmarks, - quickmarks=quickmarks) - return 'text/html', html + src = jinja.render('bookmarks.html', + title='Bookmarks', + bookmarks=bookmarks, + quickmarks=quickmarks) + return 'text/html', src @add_handler('tabs') @@ -218,10 +226,10 @@ def qute_tabs(_url): urlstr = tab.url().toDisplayString() tabs[str(win_id)].append((tab.title(), urlstr)) - html = jinja.render('tabs.html', - title='Tabs', - tab_list_by_window=tabs) - return 'text/html', html + src = jinja.render('tabs.html', + title='Tabs', + tab_list_by_window=tabs) + return 'text/html', src def history_data(start_time, offset=None): @@ -241,8 +249,9 @@ def history_data(start_time, offset=None): end_time = start_time - 24*60*60 entries = hist.entries_between(end_time, start_time) - return [{"url": e.url, "title": e.title or e.url, "time": e.atime} - for e in entries] + return [{"url": e.url, + "title": html.escape(e.title) or html.escape(e.url), + "time": e.atime} for e in entries] @add_handler('history') @@ -287,25 +296,25 @@ def qute_javascript(url): @add_handler('pyeval') def qute_pyeval(_url): """Handler for qute://pyeval.""" - html = jinja.render('pre.html', title='pyeval', content=pyeval_output) - return 'text/html', html + src = jinja.render('pre.html', title='pyeval', content=pyeval_output) + return 'text/html', src @add_handler('spawn-output') def qute_spawn_output(_url): """Handler for qute://spawn-output.""" - html = jinja.render('pre.html', title='spawn output', content=spawn_output) - return 'text/html', html + src = jinja.render('pre.html', title='spawn output', content=spawn_output) + return 'text/html', src @add_handler('version') @add_handler('verizon') def qute_version(_url): """Handler for qute://version.""" - html = jinja.render('version.html', title='Version info', - version=version.version(), - copyright=qutebrowser.__copyright__) - return 'text/html', html + src = jinja.render('version.html', title='Version info', + version=version.version(), + copyright=qutebrowser.__copyright__) + return 'text/html', src @add_handler('plainlog') @@ -323,8 +332,8 @@ def qute_plainlog(url): if not level: level = 'vdebug' text = log.ram_handler.dump_log(html=False, level=level) - html = jinja.render('pre.html', title='log', content=text) - return 'text/html', html + src = jinja.render('pre.html', title='log', content=text) + return 'text/html', src @add_handler('log') @@ -343,8 +352,8 @@ def qute_log(url): level = 'vdebug' html_log = log.ram_handler.dump_log(html=True, level=level) - html = jinja.render('log.html', title='log', content=html_log) - return 'text/html', html + src = jinja.render('log.html', title='log', content=html_log) + return 'text/html', src @add_handler('gpl') @@ -415,12 +424,12 @@ def qute_help(url): @add_handler('backend-warning') def qute_backend_warning(_url): """Handler for qute://backend-warning.""" - html = jinja.render('backend-warning.html', - distribution=version.distribution(), - Distribution=version.Distribution, - version=pkg_resources.parse_version, - title="Legacy backend warning") - return 'text/html', html + src = jinja.render('backend-warning.html', + distribution=version.distribution(), + Distribution=version.Distribution, + version=pkg_resources.parse_version, + title="Legacy backend warning") + return 'text/html', src def _qute_settings_set(url): @@ -447,13 +456,30 @@ def _qute_settings_set(url): @add_handler('settings') def qute_settings(url): """Handler for qute://settings. View/change qute configuration.""" + global csrf_token + if url.path() == '/set': + if url.password() != csrf_token: + message.error("Invalid CSRF token for qute://settings!") + raise QuteSchemeError("Invalid CSRF token!", + QNetworkReply.ContentAccessDenied) return _qute_settings_set(url) - html = jinja.render('settings.html', title='settings', - configdata=configdata, - confget=config.instance.get_str) - return 'text/html', html + # Requests to qute://settings/set should only be allowed from + # qute://settings. As an additional security precaution, we generate a CSRF + # token to use here. + if secrets: + csrf_token = secrets.token_urlsafe() + else: + # On Python < 3.6, from secrets.py + token = base64.urlsafe_b64encode(os.urandom(32)) + csrf_token = token.rstrip(b'=').decode('ascii') + + src = jinja.render('settings.html', title='settings', + configdata=configdata, + confget=config.instance.get_str, + csrf_token=csrf_token) + return 'text/html', src @add_handler('bindings') @@ -467,9 +493,9 @@ def qute_bindings(_url): for mode in modes: bindings[mode] = config.key_instance.get_bindings_for(mode) - html = jinja.render('bindings.html', title='Bindings', - bindings=bindings) - return 'text/html', html + src = jinja.render('bindings.html', title='Bindings', + bindings=bindings) + return 'text/html', src @add_handler('back') @@ -478,10 +504,10 @@ def qute_back(url): Simple page to free ram / lazy load a site, goes back on focusing the tab. """ - html = jinja.render( + src = jinja.render( 'back.html', title='Suspended: ' + urllib.parse.unquote(url.fragment())) - return 'text/html', html + return 'text/html', src @add_handler('configdiff') diff --git a/qutebrowser/browser/shared.py b/qutebrowser/browser/shared.py index 8ebbe3926..2398ca2e4 100644 --- a/qutebrowser/browser/shared.py +++ b/qutebrowser/browser/shared.py @@ -34,21 +34,22 @@ class CallSuper(Exception): """Raised when the caller should call the superclass instead.""" -def custom_headers(): +def custom_headers(url): """Get the combined custom headers.""" headers = {} - dnt_config = config.val.content.headers.do_not_track + dnt_config = config.instance.get('content.headers.do_not_track', url=url) if dnt_config is not None: dnt = b'1' if dnt_config else b'0' headers[b'DNT'] = dnt headers[b'X-Do-Not-Track'] = dnt - conf_headers = config.val.content.headers.custom + conf_headers = config.instance.get('content.headers.custom', url=url) for header, value in conf_headers.items(): headers[header.encode('ascii')] = value.encode('ascii') - accept_language = config.val.content.headers.accept_language + accept_language = config.instance.get('content.headers.accept_language', + url=url) if accept_language is not None: headers[b'Accept-Language'] = accept_language.encode('ascii') @@ -156,7 +157,7 @@ def ignore_certificate_errors(url, errors, abort_on): Return: True if the error should be ignored, False otherwise. """ - ssl_strict = config.val.content.ssl_strict + ssl_strict = config.instance.get('content.ssl_strict', url=url) log.webview.debug("Certificate errors {!r}, strict {}".format( errors, ssl_strict)) @@ -213,7 +214,7 @@ def feature_permission(url, option, msg, yes_action, no_action, abort_on, The Question object if a question was asked (and blocking=False), None otherwise. """ - config_val = config.instance.get(option) + config_val = config.instance.get(option, url=url) if config_val == 'ask': if url.isValid(): urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded) @@ -312,10 +313,10 @@ def netrc_authentication(url, authenticator): (user, _account, password) = authenticators except FileNotFoundError: log.misc.debug("No .netrc file found") - except OSError: - log.misc.exception("Unable to read the netrc file") - except netrc.NetrcParseError: - log.misc.exception("Error when parsing the netrc file") + except OSError as e: + log.misc.exception("Unable to read the netrc file: {}".format(e)) + except netrc.NetrcParseError as e: + log.misc.exception("Error when parsing the netrc file: {}".format(e)) if user is None: return False diff --git a/qutebrowser/browser/urlmarks.py b/qutebrowser/browser/urlmarks.py index 0a0dfb4f2..bb20e2166 100644 --- a/qutebrowser/browser/urlmarks.py +++ b/qutebrowser/browser/urlmarks.py @@ -240,8 +240,7 @@ class BookmarkManager(UrlMarkManager): def _init_lineparser(self): bookmarks_directory = os.path.join(standarddir.config(), 'bookmarks') - if not os.path.isdir(bookmarks_directory): - os.makedirs(bookmarks_directory) + os.makedirs(bookmarks_directory, exist_ok=True) bookmarks_subdir = os.path.join('bookmarks', 'urls') self._lineparser = lineparser.LineParser( diff --git a/qutebrowser/browser/webengine/certificateerror.py b/qutebrowser/browser/webengine/certificateerror.py index 47953d4cc..768f54ec6 100644 --- a/qutebrowser/browser/webengine/certificateerror.py +++ b/qutebrowser/browser/webengine/certificateerror.py @@ -28,6 +28,10 @@ class CertificateErrorWrapper(usertypes.AbstractCertificateErrorWrapper): """A wrapper over a QWebEngineCertificateError.""" + def __init__(self, error): + super().__init__(error) + self.ignore = False + def __str__(self): return self._error.errorDescription() @@ -37,5 +41,8 @@ class CertificateErrorWrapper(usertypes.AbstractCertificateErrorWrapper): self._error.error()), string=str(self)) + def url(self): + return self._error.url() + def is_overridable(self): return self._error.isOverridable() diff --git a/qutebrowser/browser/webengine/interceptor.py b/qutebrowser/browser/webengine/interceptor.py index b04b7962e..4ea27adf3 100644 --- a/qutebrowser/browser/webengine/interceptor.py +++ b/qutebrowser/browser/webengine/interceptor.py @@ -19,6 +19,7 @@ """A request interceptor taking care of adblocking and custom headers.""" +from PyQt5.QtCore import QUrl from PyQt5.QtWebEngineCore import (QWebEngineUrlRequestInterceptor, QWebEngineUrlRequestInfo) @@ -68,15 +69,29 @@ class RequestInterceptor(QWebEngineUrlRequestInterceptor): info.firstPartyUrl().toDisplayString(), resource_type, navigation_type)) + url = info.requestUrl() + firstparty = info.firstPartyUrl() + + if ((url.scheme(), url.host(), url.path()) == + ('qute', 'settings', '/set')): + if (firstparty != QUrl('qute://settings/') or + info.resourceType() != + QWebEngineUrlRequestInfo.ResourceTypeXhr): + log.webview.warning("Blocking malicious request from {} to {}" + .format(firstparty.toDisplayString(), + url.toDisplayString())) + info.block(True) + return + # FIXME:qtwebengine only block ads for NavigationTypeOther? - if self._host_blocker.is_blocked(info.requestUrl()): + if self._host_blocker.is_blocked(url): log.webview.info("Request to {} blocked by host blocker.".format( - info.requestUrl().host())) + url.host())) info.block(True) - for header, value in shared.custom_headers(): + for header, value in shared.custom_headers(url=url): info.setHttpHeader(header, value) - user_agent = config.val.content.headers.user_agent + user_agent = config.instance.get('content.headers.user_agent', url=url) if user_agent is not None: info.setHttpHeader(b'User-Agent', user_agent.encode('ascii')) diff --git a/qutebrowser/browser/webengine/spell.py b/qutebrowser/browser/webengine/spell.py index 5c2ed551b..55d8aea5b 100644 --- a/qutebrowser/browser/webengine/spell.py +++ b/qutebrowser/browser/webengine/spell.py @@ -21,10 +21,12 @@ import glob import os +import os.path import re +import shutil from PyQt5.QtCore import QLibraryInfo -from qutebrowser.utils import log, message +from qutebrowser.utils import log, message, standarddir, qtutils dict_version_re = re.compile(r".+-(?P[0-9]+-[0-9]+?)\.bdic") @@ -39,9 +41,12 @@ def version(filename): return tuple(int(n) for n in match.group('version').split('-')) -def dictionary_dir(): +def dictionary_dir(old=False): """Return the path (str) to the QtWebEngine's dictionaries directory.""" - datapath = QLibraryInfo.location(QLibraryInfo.DataPath) + if qtutils.version_check('5.10', compiled=False) and not old: + datapath = standarddir.data() + else: + datapath = QLibraryInfo.location(QLibraryInfo.DataPath) return os.path.join(datapath, 'qtwebengine_dictionaries') @@ -73,3 +78,16 @@ def local_filename(code): """ all_installed = local_files(code) return os.path.splitext(all_installed[0])[0] if all_installed else None + + +def init(): + """Initialize the dictionary path if supported.""" + if qtutils.version_check('5.10', compiled=False): + new_dir = dictionary_dir() + old_dir = dictionary_dir(old=True) + os.environ['QTWEBENGINE_DICTIONARIES_PATH'] = new_dir + try: + if os.path.exists(old_dir) and not os.path.exists(new_dir): + shutil.copytree(old_dir, new_dir) + except OSError: + log.misc.exception("Failed to copy old dictionaries") diff --git a/qutebrowser/browser/webengine/webengineelem.py b/qutebrowser/browser/webengine/webengineelem.py index 01ec7b687..aefa51f5a 100644 --- a/qutebrowser/browser/webengine/webengineelem.py +++ b/qutebrowser/browser/webengine/webengineelem.py @@ -203,6 +203,8 @@ class WebEngineElement(webelem.AbstractWebElement): url = self.resolve_url(baseurl) if url is None: return True + if baseurl.scheme() == url.scheme(): # e.g. a qute:// link + return False return url.scheme() not in urlutils.WEBENGINE_SCHEMES def _click_editable(self, click_target): diff --git a/qutebrowser/browser/webengine/webenginequtescheme.py b/qutebrowser/browser/webengine/webenginequtescheme.py index 12ab6af31..3eb7c7df1 100644 --- a/qutebrowser/browser/webengine/webenginequtescheme.py +++ b/qutebrowser/browser/webengine/webenginequtescheme.py @@ -37,6 +37,7 @@ class QuteSchemeHandler(QWebEngineUrlSchemeHandler): if qtutils.version_check('5.11', compiled=False): # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-63378 profile.installUrlSchemeHandler(b'chrome-error', self) + profile.installUrlSchemeHandler(b'chrome-extension', self) def requestStarted(self, job): """Handle a request for a qute: scheme. @@ -49,13 +50,33 @@ class QuteSchemeHandler(QWebEngineUrlSchemeHandler): """ url = job.requestUrl() - if url.scheme() == 'chrome-error': + if url.scheme() in ['chrome-error', 'chrome-extension']: # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-63378 job.fail(QWebEngineUrlRequestJob.UrlInvalid) return - assert job.requestMethod() == b'GET' + # Only the browser itself or qute:// pages should access any of those + # URLs. + # The request interceptor further locks down qute://settings/set. + try: + initiator = job.initiator() + except AttributeError: + # Added in Qt 5.11 + pass + else: + if initiator.isValid() and initiator.scheme() != 'qute': + log.misc.warning("Blocking malicious request from {} to {}" + .format(initiator.toDisplayString(), + url.toDisplayString())) + job.fail(QWebEngineUrlRequestJob.RequestDenied) + return + + if job.requestMethod() != b'GET': + job.fail(QWebEngineUrlRequestJob.RequestDenied) + return + assert url.scheme() == 'qute' + log.misc.debug("Got request for {}".format(url.toDisplayString())) try: mimetype, data = qutescheme.data_for_url(url) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index bae8aaffb..77f82526a 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -166,6 +166,8 @@ class WebEngineSettings(websettings.AbstractSettings): # Qt 5.11 'content.autoplay': ('PlaybackRequiresUserGesture', lambda val: not val), + 'content.webrtc_public_interfaces_only': + ('WebRTCPublicInterfacesOnly', None), } for name, (attribute, converter) in new_attributes.items(): try: @@ -298,6 +300,8 @@ def init(args): not hasattr(QWebEnginePage, 'setInspectedPage')): # only Qt < 5.11 os.environ['QTWEBENGINE_REMOTE_DEBUGGING'] = str(utils.random_port()) + spell.init() + _init_profiles() config.instance.changed.connect(_update_settings) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 86f5e08c8..cfb809097 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -25,7 +25,6 @@ import sys import re import html as html_utils -import sip from PyQt5.QtCore import (pyqtSignal, pyqtSlot, Qt, QEvent, QPoint, QPointF, QUrl, QTimer, QObject, qVersion) from PyQt5.QtGui import QKeyEvent, QIcon @@ -34,14 +33,15 @@ from PyQt5.QtWidgets import QApplication from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineScript from qutebrowser.config import configdata, config -from qutebrowser.browser import browsertab, mouse, shared +from qutebrowser.browser import browsertab, mouse, shared, webelem from qutebrowser.browser.webengine import (webview, webengineelem, tabhistory, interceptor, webenginequtescheme, cookies, webenginedownloads, - webenginesettings) + webenginesettings, certificateerror) from qutebrowser.misc import miscwidgets from qutebrowser.utils import (usertypes, qtutils, log, javascript, utils, message, objreg, jinja, debug) +from qutebrowser.qt import sip _qute_scheme_handler = None @@ -360,7 +360,11 @@ class WebEngineCaret(browsertab.AbstractCaret): if elem.is_link(): log.webview.debug("Found link in selection, clicking. ClickTarget " "{}, elem {}".format(click_type, elem)) - elem.click(click_type) + try: + elem.click(click_type) + except webelem.Error as e: + message.error(str(e)) + return def follow_selected(self, *, tab=False): if self._tab.search.search_displayed: @@ -703,6 +707,18 @@ class _WebEnginePermissions(QObject): QWebEnginePage.MediaVideoCapture: 'record video', QWebEnginePage.MediaAudioVideoCapture: 'record audio/video', } + try: + options.update({ + QWebEnginePage.MouseLock: + 'content.mouse_lock', + }) + messages.update({ + QWebEnginePage.MouseLock: + 'hide your mouse pointer', + }) + except AttributeError: + # Added in Qt 5.8 + pass try: options.update({ QWebEnginePage.DesktopVideoCapture: @@ -788,6 +804,7 @@ class _WebEngineScripts(QObject): super().__init__(parent) self._tab = tab self._widget = None + self._greasemonkey = objreg.get('greasemonkey') def connect_signals(self): config.instance.changed.connect(self._on_config_changed) @@ -853,9 +870,16 @@ class _WebEngineScripts(QObject): self._inject_early_js('js', js_code, subframes=True) self._init_stylesheet() - greasemonkey = objreg.get('greasemonkey') - greasemonkey.scripts_reloaded.connect(self._inject_userscripts) - self._inject_userscripts() + # The Greasemonkey metadata block support in QtWebEngine only starts at + # Qt 5.8. With 5.7.1, we need to inject the scripts ourselves in + # response to urlChanged. + if not qtutils.version_check('5.8'): + self._tab.url_changed.connect( + self._inject_greasemonkey_scripts_for_url) + else: + self._greasemonkey.scripts_reloaded.connect( + self._inject_all_greasemonkey_scripts) + self._inject_all_greasemonkey_scripts() def _init_stylesheet(self): """Initialize custom stylesheets. @@ -872,40 +896,77 @@ class _WebEngineScripts(QObject): ) self._inject_early_js('stylesheet', js_code, subframes=True) - def _inject_userscripts(self): - """Register user JavaScript files with the global profiles.""" - # The Greasemonkey metadata block support in QtWebEngine only starts at - # Qt 5.8. With 5.7.1, we need to inject the scripts ourselves in - # response to urlChanged. - if not qtutils.version_check('5.8'): + @pyqtSlot(QUrl) + def _inject_greasemonkey_scripts_for_url(self, url): + matching_scripts = self._greasemonkey.scripts_for(url) + self._inject_greasemonkey_scripts( + matching_scripts.start, QWebEngineScript.DocumentCreation, True) + self._inject_greasemonkey_scripts( + matching_scripts.end, QWebEngineScript.DocumentReady, False) + self._inject_greasemonkey_scripts( + matching_scripts.idle, QWebEngineScript.Deferred, False) + + @pyqtSlot() + def _inject_all_greasemonkey_scripts(self): + scripts = self._greasemonkey.all_scripts() + self._inject_greasemonkey_scripts(scripts) + + def _inject_greasemonkey_scripts(self, scripts=None, injection_point=None, + remove_first=True): + """Register user JavaScript files with the current tab. + + Args: + scripts: A list of GreasemonkeyScripts, or None to add all + known by the Greasemonkey subsystem. + injection_point: The QWebEngineScript::InjectionPoint stage + to inject the script into, None to use + auto-detection. + remove_first: Whether to remove all previously injected + scripts before adding these ones. + """ + if sip.isdeleted(self._widget): return - # Since we are inserting scripts into profile.scripts they won't - # just get replaced by new gm scripts like if we were injecting them - # ourselves so we need to remove all gm scripts, while not removing - # any other stuff that might have been added. Like the one for - # stylesheets. - greasemonkey = objreg.get('greasemonkey') - scripts = self._widget.page().scripts() - for script in scripts.toList(): - if script.name().startswith("GM-"): - log.greasemonkey.debug('Removing script: {}' - .format(script.name())) - removed = scripts.remove(script) - assert removed, script.name() + # Since we are inserting scripts into a per-tab collection, + # rather than just injecting scripts on page load, we need to + # make sure we replace existing scripts, not just add new ones. + # While, taking care not to remove any other scripts that might + # have been added elsewhere, like the one for stylesheets. + page_scripts = self._widget.page().scripts() + if remove_first: + for script in page_scripts.toList(): + if script.name().startswith("GM-"): + log.greasemonkey.debug('Removing script: {}' + .format(script.name())) + removed = page_scripts.remove(script) + assert removed, script.name() - # Then add the new scripts. - for script in greasemonkey.all_scripts(): - # @run-at (and @include/@exclude/@match) is parsed by - # QWebEngineScript. + if not scripts: + return + + for script in scripts: new_script = QWebEngineScript() - new_script.setWorldId(QWebEngineScript.MainWorld) + try: + world = int(script.jsworld) + except ValueError: + try: + world = _JS_WORLD_MAP[usertypes.JsWorld[ + script.jsworld.lower()]] + except KeyError: + log.greasemonkey.error( + "script {} has invalid value for '@qute-js-world'" + ": {}".format(script.name, script.jsworld)) + continue + new_script.setWorldId(world) new_script.setSourceCode(script.code()) new_script.setName("GM-{}".format(script.name)) new_script.setRunsOnSubFrames(script.runs_on_sub_frames) + # Override the @run-at value parsed by QWebEngineScript if desired. + if injection_point: + new_script.setInjectionPoint(injection_point) log.greasemonkey.debug('adding script: {}' .format(new_script.name())) - scripts.insert(new_script) + page_scripts.insert(new_script) class WebEngineTab(browsertab.AbstractTab): @@ -931,7 +992,7 @@ class WebEngineTab(browsertab.AbstractTab): tab=self, parent=self) self.zoom = WebEngineZoom(tab=self, parent=self) self.search = WebEngineSearch(parent=self) - self.printing = WebEnginePrinting() + self.printing = WebEnginePrinting(tab=self) self.elements = WebEngineElements(tab=self) self.action = WebEngineAction(tab=self) self.audio = WebEngineAudio(parent=self) @@ -979,6 +1040,9 @@ class WebEngineTab(browsertab.AbstractTab): url: The QUrl to open. predict: If set to False, predicted_navigation is not emitted. """ + if sip.isdeleted(self._widget): + # https://github.com/qutebrowser/qutebrowser/issues/3896 + return self._saved_zoom = self.zoom.factor() self._openurl_prepare(url, predict=predict) self._widget.load(url) @@ -1137,11 +1201,10 @@ class WebEngineTab(browsertab.AbstractTab): @pyqtSlot() def _on_load_started(self): """Clear search when a new load is started if needed.""" - if (qtutils.version_check('5.9', compiled=False) and - not qtutils.version_check('5.9.2', compiled=False)): - # WORKAROUND for - # https://bugreports.qt.io/browse/QTBUG-61506 - self.search.clear() + # WORKAROUND for + # https://bugreports.qt.io/browse/QTBUG-61506 + # (seems to be back in later Qt versions as well) + self.search.clear() super()._on_load_started() self.data.netrc_used = False @@ -1225,6 +1288,34 @@ class WebEngineTab(browsertab.AbstractTab): # the old icon is still displayed. self.icon_changed.emit(QIcon()) + @pyqtSlot(certificateerror.CertificateErrorWrapper) + def _on_ssl_errors(self, error): + self._has_ssl_errors = True + + url = error.url() + log.webview.debug("Certificate error: {}".format(error)) + + if error.is_overridable(): + error.ignore = shared.ignore_certificate_errors( + url, [error], abort_on=[self.shutting_down, self.load_started]) + else: + log.webview.error("Non-overridable certificate error: " + "{}".format(error)) + + log.webview.debug("ignore {}, URL {}, requested {}".format( + error.ignore, url, self.url(requested=True))) + + # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-56207 + # We can't really know when to show an error page, as the error might + # have happened when loading some resource. + # However, self.url() is not available yet and the requested URL + # might not match the URL we get from the error - so we just apply a + # heuristic here. + if (not qtutils.version_check('5.9') and + not error.ignore and + url.matches(self.url(requested=True), QUrl.RemoveScheme)): + self._show_error_page(url, str(error)) + @pyqtSlot(QUrl) def _on_predicted_navigation(self, url): """If we know we're going to visit an URL soon, change the settings. @@ -1240,10 +1331,10 @@ class WebEngineTab(browsertab.AbstractTab): super()._on_navigation_request(navigation) if navigation.url == QUrl('qute://print'): - command_dispatcher = objreg.get('command-dispatcher', - scope='window', - window=self.win_id) - command_dispatcher.printpage() + try: + self.printing.show_dialog() + except browsertab.WebTabError as e: + message.error(str(e)) navigation.accepted = False if not navigation.accepted or not navigation.is_main_frame: diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index 8fce9edb9..b10cc5f9a 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -19,18 +19,17 @@ """The main browser widget for QtWebEngine.""" -import sip -from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, PYQT_VERSION +from PyQt5.QtCore import pyqtSignal, QUrl, PYQT_VERSION from PyQt5.QtGui import QPalette -from PyQt5.QtQuickWidgets import QQuickWidget -from PyQt5.QtWebEngineWidgets import (QWebEngineView, QWebEnginePage, - QWebEngineScript) +from PyQt5.QtWidgets import QWidget +from PyQt5.QtWebEngineWidgets import QWebEngineView, QWebEnginePage from qutebrowser.browser import shared -from qutebrowser.browser.webengine import certificateerror, webenginesettings +from qutebrowser.browser.webengine import webenginesettings, certificateerror from qutebrowser.config import config -from qutebrowser.utils import log, debug, usertypes, jinja, objreg, qtutils +from qutebrowser.utils import log, debug, usertypes, objreg, qtutils from qutebrowser.misc import miscwidgets +from qutebrowser.qt import sip class WebEngineView(QWebEngineView): @@ -71,10 +70,10 @@ class WebEngineView(QWebEngineView): if proxy is not None: return proxy - # This should only find the RenderWidgetHostViewQtDelegateWidget, - # but not e.g. a QMenu - children = [c for c in self.findChildren(QQuickWidget) - if c.isVisible()] + # We don't want e.g. a QMenu. + rwhv_class = 'QtWebEngineCore::RenderWidgetHostViewQtDelegateWidget' + children = [c for c in self.findChildren(QWidget) + if c.isVisible() and c.inherits(rwhv_class)] log.webview.debug("Found possibly lost focusProxy: {}" .format(children)) @@ -152,11 +151,13 @@ class WebEnginePage(QWebEnginePage): Signals: certificate_error: Emitted on certificate errors. + Needs to be directly connected to a slot setting the + 'ignore' attribute. shutting_down: Emitted when the page is shutting down. navigation_request: Emitted on acceptNavigationRequest. """ - certificate_error = pyqtSignal() + certificate_error = pyqtSignal(certificateerror.CertificateErrorWrapper) shutting_down = pyqtSignal() navigation_request = pyqtSignal(usertypes.NavigationRequest) @@ -166,7 +167,6 @@ class WebEnginePage(QWebEnginePage): self._theme_color = theme_color self._set_bg_color() config.instance.changed.connect(self._set_bg_color) - self.urlChanged.connect(self._inject_userjs) @config.change_filter('colors.webpage.bg') def _set_bg_color(self): @@ -181,36 +181,9 @@ class WebEnginePage(QWebEnginePage): def certificateError(self, error): """Handle certificate errors coming from Qt.""" - self.certificate_error.emit() - url = error.url() error = certificateerror.CertificateErrorWrapper(error) - log.webview.debug("Certificate error: {}".format(error)) - - url_string = url.toDisplayString() - error_page = jinja.render( - 'error.html', title="Error loading page: {}".format(url_string), - url=url_string, error=str(error)) - - if error.is_overridable(): - ignore = shared.ignore_certificate_errors( - url, [error], abort_on=[self.loadStarted, self.shutting_down]) - else: - log.webview.error("Non-overridable certificate error: " - "{}".format(error)) - ignore = False - - # We can't really know when to show an error page, as the error might - # have happened when loading some resource. - # However, self.url() is not available yet and self.requestedUrl() - # might not match the URL we get from the error - so we just apply a - # heuristic here. - # See https://bugreports.qt.io/browse/QTBUG-56207 - log.webview.debug("ignore {}, URL {}, requested {}".format( - ignore, url, self.requestedUrl())) - if not ignore and url.matches(self.requestedUrl(), QUrl.RemoveScheme): - self.setHtml(error_page) - - return ignore + self.certificate_error.emit(error) + return error.ignore def javaScriptConfirm(self, url, js_msg): """Override javaScriptConfirm to use qutebrowser prompts.""" @@ -288,43 +261,3 @@ class WebEnginePage(QWebEnginePage): is_main_frame=is_main_frame) self.navigation_request.emit(navigation) return navigation.accepted - - @pyqtSlot('QUrl') - def _inject_userjs(self, url): - """Inject userscripts registered for `url` into the current page.""" - if qtutils.version_check('5.8'): - # Handled in webenginetab with the builtin Greasemonkey - # support. - return - - # Using QWebEnginePage.scripts() to hold the user scripts means - # we don't have to worry ourselves about where to inject the - # page but also means scripts hang around for the tab lifecycle. - # So clear them here. - scripts = self.scripts() - for script in scripts.toList(): - if script.name().startswith("GM-"): - log.greasemonkey.debug("Removing script: {}" - .format(script.name())) - removed = scripts.remove(script) - assert removed, script.name() - - def _add_script(script, injection_point): - new_script = QWebEngineScript() - new_script.setInjectionPoint(injection_point) - new_script.setWorldId(QWebEngineScript.MainWorld) - new_script.setSourceCode(script.code()) - new_script.setName("GM-{}".format(script.name)) - new_script.setRunsOnSubFrames(script.runs_on_sub_frames) - log.greasemonkey.debug("Adding script: {}" - .format(new_script.name())) - scripts.insert(new_script) - - greasemonkey = objreg.get('greasemonkey') - matching_scripts = greasemonkey.scripts_for(url) - for script in matching_scripts.start: - _add_script(script, QWebEngineScript.DocumentCreation) - for script in matching_scripts.end: - _add_script(script, QWebEngineScript.DocumentReady) - for script in matching_scripts.idle: - _add_script(script, QWebEngineScript.Deferred) diff --git a/qutebrowser/browser/webkit/mhtml.py b/qutebrowser/browser/webkit/mhtml.py index 5f495274a..9339df9ce 100644 --- a/qutebrowser/browser/webkit/mhtml.py +++ b/qutebrowser/browser/webkit/mhtml.py @@ -312,9 +312,9 @@ class _Downloader: for style in styles: style = webkitelem.WebKitElement(style, tab=self.tab) # The Mozilla Developer Network says: - # type: This attribute defines the styling language as a MIME type - # (charset should not be specified). This attribute is optional and - # default to text/css if it's missing. + # > type: This attribute defines the styling language as a MIME + # > type (charset should not be specified). This attribute is + # > optional and default to text/css if it's missing. # https://developer.mozilla.org/en/docs/Web/HTML/Element/style if 'type' in style and style['type'] != 'text/css': continue diff --git a/qutebrowser/browser/webkit/network/filescheme.py b/qutebrowser/browser/webkit/network/filescheme.py index 840ed6a4a..a29674e25 100644 --- a/qutebrowser/browser/webkit/network/filescheme.py +++ b/qutebrowser/browser/webkit/network/filescheme.py @@ -111,11 +111,13 @@ def dirbrowser_html(path): return html.encode('UTF-8', errors='xmlcharrefreplace') -def handler(request): +def handler(request, _operation, _current_url): """Handler for a file:// URL. Args: request: QNetworkRequest to answer to. + _operation: The HTTP operation being done. + _current_url: The page we're on currently. Return: A QNetworkReply for directories, None for files. diff --git a/qutebrowser/browser/webkit/network/networkmanager.py b/qutebrowser/browser/webkit/network/networkmanager.py index a66802375..0406f8bdf 100644 --- a/qutebrowser/browser/webkit/network/networkmanager.py +++ b/qutebrowser/browser/webkit/network/networkmanager.py @@ -373,14 +373,7 @@ class NetworkManager(QNetworkAccessManager): req, proxy_error, QNetworkReply.UnknownProxyError, self) - scheme = req.url().scheme() - if scheme in self._scheme_handlers: - result = self._scheme_handlers[scheme](req) - if result is not None: - result.setParent(self) - return result - - for header, value in shared.custom_headers(): + for header, value in shared.custom_headers(url=req.url()): req.setRawHeader(header, value) host_blocker = objreg.get('host-blocker') @@ -416,5 +409,12 @@ class NetworkManager(QNetworkAccessManager): req.url().toDisplayString(), current_url.toDisplayString())) + scheme = req.url().scheme() + if scheme in self._scheme_handlers: + result = self._scheme_handlers[scheme](req, op, current_url) + if result is not None: + result.setParent(self) + return result + self.set_referer(req, current_url) return super().createRequest(op, req, outgoing_data) diff --git a/qutebrowser/browser/webkit/network/webkitqutescheme.py b/qutebrowser/browser/webkit/network/webkitqutescheme.py index d732b6ab0..b6f99437a 100644 --- a/qutebrowser/browser/webkit/network/webkitqutescheme.py +++ b/qutebrowser/browser/webkit/network/webkitqutescheme.py @@ -21,27 +21,46 @@ import mimetypes -from PyQt5.QtNetwork import QNetworkReply +from PyQt5.QtCore import QUrl +from PyQt5.QtNetwork import QNetworkReply, QNetworkAccessManager from qutebrowser.browser import pdfjs, qutescheme from qutebrowser.browser.webkit.network import networkreply from qutebrowser.utils import log, usertypes, qtutils -def handler(request): +def handler(request, operation, current_url): """Scheme handler for qute:// URLs. Args: request: QNetworkRequest to answer to. + operation: The HTTP operation being done. + current_url: The page we're on currently. Return: A QNetworkReply. """ + if operation != QNetworkAccessManager.GetOperation: + return networkreply.ErrorNetworkReply( + request, "Unsupported request type", + QNetworkReply.ContentOperationNotPermittedError) + + url = request.url() + + if ((url.scheme(), url.host(), url.path()) == + ('qute', 'settings', '/set')): + if current_url != QUrl('qute://settings/'): + log.webview.warning("Blocking malicious request from {} to {}" + .format(current_url.toDisplayString(), + url.toDisplayString())) + return networkreply.ErrorNetworkReply( + request, "Invalid qute://settings request", + QNetworkReply.ContentAccessDenied) + try: - mimetype, data = qutescheme.data_for_url(request.url()) + mimetype, data = qutescheme.data_for_url(url) except qutescheme.NoHandlerFound: - errorstr = "No handler found for {}!".format( - request.url().toDisplayString()) + errorstr = "No handler found for {}!".format(url.toDisplayString()) return networkreply.ErrorNetworkReply( request, errorstr, QNetworkReply.ContentNotFoundError) except qutescheme.QuteSchemeOSError as e: diff --git a/qutebrowser/browser/webkit/webkittab.py b/qutebrowser/browser/webkit/webkittab.py index cf158ed2d..7b7ad0c7d 100644 --- a/qutebrowser/browser/webkit/webkittab.py +++ b/qutebrowser/browser/webkit/webkittab.py @@ -23,7 +23,6 @@ import re import functools import xml.etree.ElementTree -import sip from PyQt5.QtCore import (pyqtSlot, Qt, QEvent, QUrl, QPoint, QTimer, QSizeF, QSize) from PyQt5.QtGui import QKeyEvent, QIcon @@ -35,6 +34,7 @@ from qutebrowser.browser import browsertab, shared from qutebrowser.browser.webkit import (webview, tabhistory, webkitelem, webkitsettings) from qutebrowser.utils import qtutils, usertypes, utils, log, debug +from qutebrowser.qt import sip class WebKitAction(browsertab.AbstractAction): @@ -658,7 +658,7 @@ class WebKitTab(browsertab.AbstractTab): tab=self, parent=self) self.zoom = WebKitZoom(tab=self, parent=self) self.search = WebKitSearch(parent=self) - self.printing = WebKitPrinting() + self.printing = WebKitPrinting(tab=self) self.elements = WebKitElements(tab=self) self.action = WebKitAction(tab=self) self.audio = WebKitAudio(parent=self) @@ -808,6 +808,10 @@ class WebKitTab(browsertab.AbstractTab): if navigation.is_main_frame: self.settings.update_for_url(navigation.url) + @pyqtSlot() + def _on_ssl_errors(self): + self._has_ssl_errors = True + def _connect_signals(self): view = self._widget page = view.page() diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 6bbc27109..4e0701329 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -22,7 +22,6 @@ import html import functools -import sip from PyQt5.QtCore import pyqtSlot, pyqtSignal, Qt, QUrl, QPoint from PyQt5.QtGui import QDesktopServices from PyQt5.QtNetwork import QNetworkReply, QNetworkRequest @@ -35,6 +34,7 @@ from qutebrowser.browser import pdfjs, shared from qutebrowser.browser.webkit import http from qutebrowser.browser.webkit.network import networkmanager from qutebrowser.utils import message, usertypes, log, jinja, objreg +from qutebrowser.qt import sip class BrowserPage(QWebPage): @@ -212,7 +212,8 @@ class BrowserPage(QWebPage): page = pdfjs.generate_pdfjs_page(reply.url()) except pdfjs.PDFJSNotFound: page = jinja.render('no_pdfjs.html', - url=reply.url().toDisplayString()) + url=reply.url().toDisplayString(), + title="PDF.js not found") self.mainFrame().setContent(page.encode('utf-8'), 'text/html', reply.url()) reply.deleteLater() @@ -415,7 +416,7 @@ class BrowserPage(QWebPage): def userAgentForUrl(self, url): """Override QWebPage::userAgentForUrl to customize the user agent.""" - ua = config.val.content.headers.user_agent + ua = config.instance.get('content.headers.user_agent', url=url) if ua is None: return super().userAgentForUrl(url) else: diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 35beb24de..74b75aeb1 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -122,8 +122,8 @@ def _buffer(skip_win_id=None): tabs.append(("{}/{}".format(win_id, idx + 1), tab.url().toDisplayString(), tabbed_browser.widget.page_title(idx))) - cat = listcategory.ListCategory("{}".format(win_id), tabs, - delete_func=delete_buffer) + cat = listcategory.ListCategory( + str(win_id), tabs, delete_func=delete_buffer, sort=False) model.add_category(cat) return model diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 1f06fe861..708caa258 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -60,10 +60,14 @@ def url(*, info): quickmarks = [(url, name) for (name, url) in objreg.get('quickmark-manager').marks.items()] bookmarks = objreg.get('bookmark-manager').marks.items() - searchengines = config.val.url.searchengines.items() - categories = config.val.url.open_categories_shown + # pylint: disable=bad-config-option + searchengines = {k:v for k, v in config.val.url.searchengines.items() + if k not in "DEFAULT"}.items() + # pylint: enable=bad-config-option + categories = config.val.completion.open_categories models = {} + if searchengines and "searchengines" in categories: models["searchengines"] = listcategory.ListCategory( 'Search engines', searchengines, sort=False) diff --git a/qutebrowser/config/configdata.py b/qutebrowser/config/configdata.py index da3025319..dace0772a 100644 --- a/qutebrowser/config/configdata.py +++ b/qutebrowser/config/configdata.py @@ -56,7 +56,7 @@ class Option: @attr.s class Migrations: - """Nigrated options in configdata.yml. + """Migrated options in configdata.yml. Attributes: renamed: A dict mapping old option names to new names. diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index d2176916c..c6acc10e3 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -321,6 +321,7 @@ content.windowed_fullscreen: content.desktop_capture: type: BoolAsk default: ask + supports_pattern: true desc: >- Allow websites to share screen content. @@ -350,14 +351,28 @@ content.frame_flattening: content.geolocation: default: ask type: BoolAsk + supports_pattern: true desc: Allow websites to request geolocations. +content.mouse_lock: + default: ask + type: BoolAsk + supports_pattern: true + backend: + QtWebKit: false + QtWebEngine: Qt 5.8 + desc: Allow websites to lock your mouse pointer. + content.headers.accept_language: type: name: String none_ok: true + supports_pattern: true default: en-US,en - desc: Value to send in the `Accept-Language` header. + desc: >- + Value to send in the `Accept-Language` header. + + Note that the value read from JavaScript is always the global value. content.headers.custom: default: {} @@ -370,6 +385,7 @@ content.headers.custom: name: String encoding: ascii none_ok: true + supports_pattern: true desc: Custom headers for qutebrowser HTTP requests. content.headers.do_not_track: @@ -377,6 +393,7 @@ content.headers.do_not_track: name: Bool none_ok: true default: true + supports_pattern: true desc: >- Value to send in the `DNT` header. @@ -451,7 +468,11 @@ content.headers.user_agent: Gecko" - IE 11.0 for Desktop Win7 64-bit - desc: User agent to send. Unset to send the default. + supports_pattern: true + desc: >- + User agent to send. Unset to send the default. + + Note that the value read from JavaScript is always the global value. content.host_blocking.enabled: default: true @@ -594,6 +615,7 @@ content.local_storage: content.media_capture: default: ask type: BoolAsk + supports_pattern: true backend: QtWebEngine desc: Allow websites to record audio/video. @@ -610,6 +632,7 @@ content.netrc_file: content.notifications: default: ask type: BoolAsk + supports_pattern: true backend: QtWebKit desc: Allow websites to show notifications. @@ -626,6 +649,7 @@ content.pdfjs: content.persistent_storage: default: ask type: BoolAsk + supports_pattern: true backend: QtWebKit: false QtWebEngine: Qt 5.11 @@ -672,6 +696,7 @@ content.proxy_dns_requests: content.register_protocol_handler: default: ask type: BoolAsk + supports_pattern: true backend: QtWebKit: false QtWebEngine: Qt 5.11 @@ -681,6 +706,7 @@ content.register_protocol_handler: content.ssl_strict: default: ask type: BoolAsk + supports_pattern: true desc: Validate SSL handshakes. content.user_stylesheets: @@ -712,14 +738,13 @@ content.webrtc_public_interfaces_only: content.xss_auditing: type: Bool - default: false + default: true supports_pattern: true desc: >- Monitor load requests for cross-site scripting attempts. Suspicious scripts will be blocked and reported in the inspector's - JavaScript console. Enabling this feature might have an impact on - performance. + JavaScript console. # emacs: ' @@ -853,6 +878,18 @@ downloads.location.suggestion: - both: Show download path and filename. desc: What to display in the download filename input. +completion.open_categories: + type: + name: FlagList + valid_values: [searchengines, quickmarks, bookmarks, history] + none_ok: true + default: + - searchengines + - quickmarks + - bookmarks + - history + desc: Which categories to show (in which order) in the :open completion. + downloads.open_dispatcher: type: name: String @@ -1565,19 +1602,6 @@ url.open_base_url: default: false desc: Open base URL of the searchengine if a searchengine shortcut is invoked without parameters. -url.open_categories_shown: - type: - name: List - valtype: String - none_ok: true - default: - - searchengines - - quickmarks - - bookmarks - - history - desc: Which categories to show in the :open dialogue. The order of this list is used for ordering the categories. - - url.searchengines: default: DEFAULT: https://duckduckgo.com/?q={} @@ -2452,6 +2476,7 @@ bindings.default: .: repeat-command : tab-pin : tab-mute + gD: tab-give q: record-macro "@": run-macro tsh: config-cycle -p -t -u *://{url:host}/* content.javascript.enabled ;; reload @@ -2477,7 +2502,7 @@ bindings.default: : hint all tab-bg : leave-mode passthrough: - : leave-mode + : leave-mode command: : command-history-prev : command-history-next diff --git a/qutebrowser/config/configinit.py b/qutebrowser/config/configinit.py index c4418a17c..99a3ff91c 100644 --- a/qutebrowser/config/configinit.py +++ b/qutebrowser/config/configinit.py @@ -89,6 +89,8 @@ def _init_envvars(): os.environ['QT_XCB_FORCE_SOFTWARE_OPENGL'] = '1' elif software_rendering == 'qt-quick': os.environ['QT_QUICK_BACKEND'] = 'software' + elif software_rendering == 'chromium': + os.environ['QT_WEBENGINE_DISABLE_NOUVEAU_WORKAROUND'] = '1' if config.val.qt.force_platform is not None: os.environ['QT_QPA_PLATFORM'] = config.val.qt.force_platform diff --git a/qutebrowser/html/settings.html b/qutebrowser/html/settings.html index 62b424a59..d4ff4ce34 100644 --- a/qutebrowser/html/settings.html +++ b/qutebrowser/html/settings.html @@ -3,7 +3,8 @@ {% block script %} var cset = function(option, value) { // FIXME:conf we might want some error handling here? - var url = "qute://settings/set?option=" + encodeURIComponent(option); + var url = "qute://user:{{csrf_token}}@settings/set" + url += "?option=" + encodeURIComponent(option); url += "&value=" + encodeURIComponent(value); var xhr = new XMLHttpRequest(); xhr.open("GET", url); diff --git a/qutebrowser/javascript/.eslintrc.yaml b/qutebrowser/javascript/.eslintrc.yaml index 02081f7a7..9e99b0aa5 100644 --- a/qutebrowser/javascript/.eslintrc.yaml +++ b/qutebrowser/javascript/.eslintrc.yaml @@ -57,3 +57,5 @@ rules: no-ternary: "off" max-lines: "off" multiline-ternary: ["error", "always-multiline"] + max-lines-per-function: "off" + require-unicode-regexp: "off" diff --git a/qutebrowser/javascript/history.js b/qutebrowser/javascript/history.js index 417441bd9..093b95b4e 100644 --- a/qutebrowser/javascript/history.js +++ b/qutebrowser/javascript/history.js @@ -114,7 +114,7 @@ window.loadHistory = (function() { title.className = "title"; const link = document.createElement("a"); link.href = itemUrl; - link.innerHTML = itemTitle; + link.innerHTML = itemTitle; // Properly escaped in qutescheme.py const host = document.createElement("span"); host.className = "hostname"; host.innerHTML = link.hostname; diff --git a/qutebrowser/keyinput/keyutils.py b/qutebrowser/keyinput/keyutils.py index f0cf9c22a..bef3c7cf0 100644 --- a/qutebrowser/keyinput/keyutils.py +++ b/qutebrowser/keyinput/keyutils.py @@ -138,6 +138,37 @@ def _key_to_string(key): 'Dead_Hook': 'Hook', 'Dead_Horn': 'Horn', + 'Dead_Stroke': '̵', + 'Dead_Abovecomma': '̓', + 'Dead_Abovereversedcomma': '̔', + 'Dead_Doublegrave': '̏', + 'Dead_Belowring': '̥', + 'Dead_Belowmacron': '̱', + 'Dead_Belowcircumflex': '̭', + 'Dead_Belowtilde': '̰', + 'Dead_Belowbreve': '̮', + 'Dead_Belowdiaeresis': '̤', + 'Dead_Invertedbreve': '̑', + 'Dead_Belowcomma': '̦', + 'Dead_Currency': '¤', + 'Dead_a': 'a', + 'Dead_A': 'A', + 'Dead_e': 'e', + 'Dead_E': 'E', + 'Dead_i': 'i', + 'Dead_I': 'I', + 'Dead_o': 'o', + 'Dead_O': 'O', + 'Dead_u': 'u', + 'Dead_U': 'U', + 'Dead_Small_Schwa': 'ə', + 'Dead_Capital_Schwa': 'Ə', + 'Dead_Greek': 'Greek', + 'Dead_Lowline': '̲', + 'Dead_Aboveverticalline': '̍', + 'Dead_Belowverticalline': '\u0329', + 'Dead_Longsolidusoverlay': '̸', + 'Memo': 'Memo', 'ToDoList': 'To Do List', 'Calendar': 'Calendar', @@ -239,8 +270,10 @@ def _parse_special_key(keystr): replacements = ( ('control', 'ctrl'), ('windows', 'meta'), - ('mod1', 'alt'), ('mod4', 'meta'), + ('command', 'meta'), + ('cmd', 'meta'), + ('mod1', 'alt'), ('less', '<'), ('greater', '>'), ) diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index 4cfee7eff..3fdad13d1 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -24,7 +24,8 @@ import base64 import itertools import functools -from PyQt5.QtCore import pyqtSlot, QRect, QPoint, QTimer, Qt +from PyQt5.QtCore import (pyqtSlot, QRect, QPoint, QTimer, Qt, + QCoreApplication, QEventLoop) from PyQt5.QtWidgets import QWidget, QVBoxLayout, QApplication, QSizePolicy from qutebrowser.commands import runners, cmdutils @@ -98,6 +99,9 @@ def raise_window(window, alert=True): window.setWindowState(window.windowState() & ~Qt.WindowMinimized) window.setWindowState(window.windowState() | Qt.WindowActive) window.raise_() + # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-69568 + QCoreApplication.processEvents( + QEventLoop.ExcludeUserInputEvents | QEventLoop.ExcludeSocketNotifiers) window.activateWindow() if alert: diff --git a/qutebrowser/mainwindow/prompt.py b/qutebrowser/mainwindow/prompt.py index f7af28440..357f63dd7 100644 --- a/qutebrowser/mainwindow/prompt.py +++ b/qutebrowser/mainwindow/prompt.py @@ -24,7 +24,6 @@ import html import collections import attr -import sip from PyQt5.QtCore import (pyqtSlot, pyqtSignal, Qt, QTimer, QDir, QModelIndex, QItemSelectionModel, QObject, QEventLoop) from PyQt5.QtWidgets import (QWidget, QGridLayout, QVBoxLayout, QLineEdit, @@ -36,6 +35,7 @@ from qutebrowser.config import config from qutebrowser.utils import usertypes, log, utils, qtutils, objreg, message from qutebrowser.keyinput import modeman from qutebrowser.commands import cmdutils, cmdexc +from qutebrowser.qt import sip prompt_queue = None diff --git a/qutebrowser/mainwindow/statusbar/url.py b/qutebrowser/mainwindow/statusbar/url.py index f24e79834..fda09d642 100644 --- a/qutebrowser/mainwindow/statusbar/url.py +++ b/qutebrowser/mainwindow/statusbar/url.py @@ -21,7 +21,7 @@ import enum -from PyQt5.QtCore import pyqtSlot, pyqtProperty, Qt, QUrl +from PyQt5.QtCore import pyqtSlot, pyqtProperty, QUrl from qutebrowser.mainwindow.statusbar import textbase from qutebrowser.config import config @@ -80,8 +80,7 @@ class UrlText(textbase.TextBase): """ def __init__(self, parent=None): - """Override TextBase.__init__ to elide in the middle by default.""" - super().__init__(parent, Qt.ElideMiddle) + super().__init__(parent) self.setObjectName(self.__class__.__name__) config.set_register_stylesheet(self) self._hover_url = None diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index 2b74df72b..9c4473874 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -506,7 +506,8 @@ class TabbedBrowser(QWidget): usertypes.KeyMode.yesno]: # If we were in a command prompt, restore old focus # The above commands need to be run to switch tabs - prev_focus.setFocus() + if prev_focus is not None: + prev_focus.setFocus() tab.show() self.new_tab.emit(tab, idx) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index ac81eba29..02934a532 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -54,6 +54,10 @@ class TabWidget(QTabWidget): tab_index_changed = pyqtSignal(int, int) new_tab_requested = pyqtSignal('QUrl', bool, bool) + # Strings for controlling the mute/audible text + MUTE_STRING = '[M] ' + AUDIBLE_STRING = '[A] ' + def __init__(self, win_id, parent=None): super().__init__(parent) bar = TabBar(win_id, self) @@ -175,9 +179,9 @@ class TabWidget(QTabWidget): fields['private'] = ' [Private Mode] ' if tab.private else '' try: if tab.audio.is_muted(): - fields['audio'] = '[M] ' + fields['audio'] = TabWidget.MUTE_STRING elif tab.audio.is_recently_audible(): - fields['audio'] = '[A] ' + fields['audio'] = TabWidget.AUDIBLE_STRING else: fields['audio'] = '' except browsertab.WebTabError: diff --git a/qutebrowser/misc/backendproblem.py b/qutebrowser/misc/backendproblem.py index b2d6f71bb..363f7f23c 100644 --- a/qutebrowser/misc/backendproblem.py +++ b/qutebrowser/misc/backendproblem.py @@ -172,6 +172,11 @@ def _nvidia_shader_workaround(): def _handle_nouveau_graphics(): + """Force software rendering when using the Nouveau driver. + + WORKAROUND for https://bugreports.qt.io/browse/QTBUG-41242 + Should be fixed in Qt 5.10 via https://codereview.qt-project.org/#/c/208664/ + """ assert objects.backend == usertypes.Backend.QtWebEngine, objects.backend if os.environ.get('QUTE_SKIP_NOUVEAU_CHECK'): @@ -181,7 +186,11 @@ def _handle_nouveau_graphics(): return if (os.environ.get('LIBGL_ALWAYS_SOFTWARE') == '1' or - 'QT_XCB_FORCE_SOFTWARE_OPENGL' in os.environ): + # qt.force_software_rendering = 'software-opengl' + 'QT_XCB_FORCE_SOFTWARE_OPENGL' in os.environ or + # qt.force_software_rendering = 'chromium', also see: + # https://build.opensuse.org/package/view_file/openSUSE:Factory/libqt5-qtwebengine/disable-gpu-when-using-nouveau-boo-1005323.diff?expand=1 + 'QT_WEBENGINE_DISABLE_NOUVEAU_WORKAROUND' in os.environ): return button = _Button("Force software rendering", 'qt.force_software_rendering', diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py index 9649d27cc..b29e1508f 100644 --- a/qutebrowser/misc/earlyinit.py +++ b/qutebrowser/misc/earlyinit.py @@ -206,6 +206,7 @@ def _check_modules(modules): # https://bitbucket.org/fdik/pypeg/commits/dd15ca462b532019c0a3be1d39b8ee2f3fa32f4e messages = ['invalid escape sequence', 'Flags not at the start of the expression'] + # pylint: disable=bad-continuation with log.ignore_py_warnings( category=DeprecationWarning, message=r'({})'.format('|'.join(messages)) @@ -216,6 +217,7 @@ def _check_modules(modules): category=ImportWarning, message=r'Not importing directory .*: missing __init__' ): + # pylint: enable=bad-continuation importlib.import_module(name) except ImportError as e: _die(text, e) @@ -246,7 +248,7 @@ def configure_pyqt(): from PyQt5.QtCore import pyqtRemoveInputHook pyqtRemoveInputHook() - import sip + from qutebrowser.qt import sip try: # Added in sip 4.19.4 sip.enableoverflowchecking(True) diff --git a/qutebrowser/misc/editor.py b/qutebrowser/misc/editor.py index 473f67c3e..038331c9b 100644 --- a/qutebrowser/misc/editor.py +++ b/qutebrowser/misc/editor.py @@ -140,11 +140,9 @@ class ExternalEditor(QObject): # the file from the external editor, see # https://github.com/qutebrowser/qutebrowser/issues/1767 with tempfile.NamedTemporaryFile( - # pylint: disable=bad-continuation mode='w', prefix=prefix, encoding=config.val.editor.encoding, delete=False) as fobj: - # pylint: enable=bad-continuation if text: fobj.write(text) return fobj.name diff --git a/qutebrowser/misc/lineparser.py b/qutebrowser/misc/lineparser.py index 6e50edb9b..e9fc67e18 100644 --- a/qutebrowser/misc/lineparser.py +++ b/qutebrowser/misc/lineparser.py @@ -72,8 +72,7 @@ class BaseLineParser(QObject): Return: True if the file should be saved, False otherwise. """ - if not os.path.exists(self._configdir): - os.makedirs(self._configdir, 0o755) + os.makedirs(self._configdir, 0o755, exist_ok=True) return True def _after_save(self): diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index dddf48b05..358ab0c53 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -24,7 +24,6 @@ import os.path import itertools import urllib -import sip from PyQt5.QtCore import QUrl, QObject, QPoint, QTimer from PyQt5.QtWidgets import QApplication import yaml @@ -35,6 +34,7 @@ from qutebrowser.commands import cmdexc, cmdutils from qutebrowser.config import config, configfiles from qutebrowser.completion.models import miscmodels from qutebrowser.mainwindow import mainwindow +from qutebrowser.qt import sip default = object() # Sentinel value diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index 9b09fb132..bf47bea5a 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -74,10 +74,10 @@ class SqliteError(SqlError): '13', # SQLITE_FULL ] # At least in init(), we can get errors like this: - # type: ConnectionError - # database text: out of memory - # driver text: Error opening database - # error code: -1 + # > type: ConnectionError + # > database text: out of memory + # > driver text: Error opening database + # > error code: -1 environmental_strings = [ "out of memory", ] diff --git a/qutebrowser/misc/utilcmds.py b/qutebrowser/misc/utilcmds.py index 5d987afa3..d108a56ac 100644 --- a/qutebrowser/misc/utilcmds.py +++ b/qutebrowser/misc/utilcmds.py @@ -29,7 +29,6 @@ try: except ImportError: hunter = None -import sip from PyQt5.QtCore import QUrl # so it's available for :debug-pyeval from PyQt5.QtWidgets import QApplication # pylint: disable=unused-import @@ -40,6 +39,7 @@ from qutebrowser.commands import cmdutils, runners, cmdexc from qutebrowser.config import config, configdata from qutebrowser.misc import consolewidget from qutebrowser.utils.version import pastebin_version +from qutebrowser.qt import sip @cmdutils.register(maxsplit=1, no_cmd_split=True, no_replace_variables=True) @@ -74,13 +74,18 @@ def later(ms: int, command, win_id): @cmdutils.register(maxsplit=1, no_cmd_split=True, no_replace_variables=True) @cmdutils.argument('win_id', win_id=True) -def repeat(times: int, command, win_id): +@cmdutils.argument('count', count=True) +def repeat(times: int, command, win_id, count=None): """Repeat a given command. Args: times: How many times to repeat. command: The command to run, with optional args. + count: Multiplies with 'times' when given. """ + if count is not None: + times *= count + if times < 0: raise cmdexc.CommandError("A negative count doesn't make sense.") commandrunner = runners.CommandRunner(win_id) @@ -365,7 +370,7 @@ def window_only(current_win_id): @cmdutils.register() def nop(): """Do nothing.""" - return + pass @cmdutils.register() diff --git a/qutebrowser/qt.py b/qutebrowser/qt.py new file mode 100644 index 000000000..2878bbe98 --- /dev/null +++ b/qutebrowser/qt.py @@ -0,0 +1,28 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2018 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Wrappers around Qt/PyQt code.""" + +# pylint: disable=unused-import +# PyQt 5.11 comes with a bundled sip, +# for older PyQt versions it's a separate module. +try: + from PyQt5 import sip +except ImportError: + import sip diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py index 30e570e16..afe0a3cb3 100644 --- a/qutebrowser/utils/log.py +++ b/qutebrowser/utils/log.py @@ -25,6 +25,7 @@ import html as pyhtml import logging import contextlib import collections +import copy import faulthandler import traceback import warnings @@ -209,6 +210,11 @@ def _init_py_warnings(): """Initialize Python warning handling.""" warnings.simplefilter('default') warnings.filterwarnings('ignore', module='pdb', category=ResourceWarning) + # This happens in many qutebrowser dependencies... + warnings.filterwarnings('ignore', category=DeprecationWarning, + message="Using or importing the ABCs from " + "'collections' instead of from 'collections.abc' " + "is deprecated, and in 3.8 it will stop working") @contextlib.contextmanager @@ -563,16 +569,14 @@ class RAMHandler(logging.Handler): https://github.com/qutebrowser/qutebrowser/issues/34 """ minlevel = LOG_LEVELS.get(level.upper(), VDEBUG_LEVEL) - lines = [] fmt = self.html_formatter.format if html else self.format self.acquire() try: - records = list(self._data) + lines = [fmt(record) + for record in self._data + if record.levelno >= minlevel] finally: self.release() - for record in records: - if record.levelno >= minlevel: - lines.append(fmt(record)) return '\n'.join(lines) def change_log_capacity(self, capacity): @@ -632,17 +636,18 @@ class HTMLFormatter(logging.Formatter): self._colordict['reset'] = '' def format(self, record): - record.__dict__.update(self._colordict) - if record.levelname in self._log_colors: - color = self._log_colors[record.levelname] - record.log_color = self._colordict[color] + record_clone = copy.copy(record) + record_clone.__dict__.update(self._colordict) + if record_clone.levelname in self._log_colors: + color = self._log_colors[record_clone.levelname] + record_clone.log_color = self._colordict[color] else: - record.log_color = '' + record_clone.log_color = '' for field in ['msg', 'filename', 'funcName', 'levelname', 'module', 'name', 'pathname', 'processName', 'threadName']: - data = str(getattr(record, field)) - setattr(record, field, pyhtml.escape(data)) - msg = super().format(record) + data = str(getattr(record_clone, field)) + setattr(record_clone, field, pyhtml.escape(data)) + msg = super().format(record_clone) if not msg.endswith(self._colordict['reset']): msg += self._colordict['reset'] return msg diff --git a/qutebrowser/utils/standarddir.py b/qutebrowser/utils/standarddir.py index 40f1fa966..95aa4d438 100644 --- a/qutebrowser/utils/standarddir.py +++ b/qutebrowser/utils/standarddir.py @@ -290,10 +290,7 @@ def _create(path): 0700. If the destination directory exists already the permissions should not be changed. """ - try: - os.makedirs(path, 0o700) - except FileExistsError: - pass + os.makedirs(path, 0o700, exist_ok=True) def _init_dirs(args=None): diff --git a/qutebrowser/utils/urlmatch.py b/qutebrowser/utils/urlmatch.py index 5d9afc13e..02db96e33 100644 --- a/qutebrowser/utils/urlmatch.py +++ b/qutebrowser/utils/urlmatch.py @@ -29,6 +29,8 @@ import ipaddress import fnmatch import urllib.parse +from PyQt5.QtCore import QUrl + from qutebrowser.utils import utils, qtutils @@ -177,6 +179,15 @@ class UrlPattern: assert self._host is None return + if parsed.netloc.startswith('['): + # Using QUrl parsing to minimize ipv6 addresses + url = QUrl() + url.setHost(parsed.hostname) + if not url.isValid(): + raise ParseError(url.errorString()) + self._host = url.host() + return + # FIXME what about multiple dots? host_parts = parsed.hostname.rstrip('.').split('.') if host_parts[0] == '*': diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py index 4554aef2e..fe255331d 100644 --- a/qutebrowser/utils/utils.py +++ b/qutebrowser/utils/utils.py @@ -645,7 +645,7 @@ def yaml_load(f): end = datetime.datetime.now() delta = (end - start).total_seconds() - deadline = 5 if 'CI' in os.environ else 2 + deadline = 10 if 'CI' in os.environ else 2 if delta > deadline: # pragma: no cover log.misc.warning( "YAML load took unusually long, please report this at " diff --git a/qutebrowser/utils/version.py b/qutebrowser/utils/version.py index dc5168a8d..b09c0ea7f 100644 --- a/qutebrowser/utils/version.py +++ b/qutebrowser/utils/version.py @@ -313,12 +313,29 @@ def _chromium_version(): http://code.qt.io/cgit/qt/qtwebengine.git/tree/tools/scripts/version_resolver.py#n41 Quick reference: + Qt 5.7: Chromium 49 + 49.0.2623.111 (2016-03-02) + 5.7.1: Security fixes up to 54.0.2840.87 (2016-10-19) + Qt 5.8: Chromium 53 + 53.0.2785.148 (2016-08-31) + 5.8.0: Security fixes up to 55.0.2883.75 (2016-12-01) + Qt 5.9: Chromium 56 + (LTS) 56.0.2924.122 (2017-01-25) + 5.9.6: Security fixes up to 66.0.3359.170 (2018-04-17) + Qt 5.10: Chromium 61 + 61.0.3163.140 (2017-09-05) + 5.10.1: Security fixes up to 64.0.3282.140 (2018-01-24) + Qt 5.11: Chromium 65 + 65.0.3325.151 (.1: .230) (2018-03-06) + 5.11.1: Security fixes up to 67.0.3396.87 (2018-05-29) + Qt 5.12: Chromium 69 (?) + current dev branch: 67.0.3396.76 (2018-05-29) Also see https://www.chromium.org/developers/calendar """ @@ -433,6 +450,11 @@ def opengl_vendor(): # pragma: no cover """ assert QApplication.instance() + override = os.environ.get('QUTE_FAKE_OPENGL_VENDOR') + if override is not None: + log.init.debug("Using override {}".format(override)) + return override + old_context = QOpenGLContext.currentContext() old_surface = None if old_context is None else old_context.surface() @@ -442,12 +464,12 @@ def opengl_vendor(): # pragma: no cover ctx = QOpenGLContext() ok = ctx.create() if not ok: - log.init.debug("opengl_vendor: Creating context failed!") + log.init.debug("Creating context failed!") return None ok = ctx.makeCurrent(surface) if not ok: - log.init.debug("opengl_vendor: Making context current failed!") + log.init.debug("Making context current failed!") return None try: @@ -461,12 +483,11 @@ def opengl_vendor(): # pragma: no cover try: vf = ctx.versionFunctions(vp) except ImportError as e: - log.init.debug("opengl_vendor: Importing version functions " - "failed: {}".format(e)) + log.init.debug("Importing version functions failed: {}".format(e)) return None if vf is None: - log.init.debug("opengl_vendor: Getting version functions failed!") + log.init.debug("Getting version functions failed!") return None return vf.glGetString(vf.GL_VENDOR) @@ -484,6 +505,7 @@ def pastebin_version(pbclient=None): def _on_paste_version_success(url): global pastebin_url + url = url.strip() _yank_url(url) pbclient.deleteLater() pastebin_url = url diff --git a/requirements.txt b/requirements.txt index 2695ba55f..b5328b5da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ Jinja2==2.10 MarkupSafe==1.0 Pygments==2.2.0 pyPEG2==2.15.2 -PyYAML==3.12 +PyYAML==3.13 diff --git a/scripts/dev/build_release.py b/scripts/dev/build_release.py index 146b7a462..254132b3c 100755 --- a/scripts/dev/build_release.py +++ b/scripts/dev/build_release.py @@ -45,7 +45,6 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, import qutebrowser from scripts import utils # from scripts.dev import update_3rdparty -from scripts.dev import gen_versioninfo def call_script(name, *args, python=sys.executable): @@ -239,31 +238,16 @@ def build_windows(): except FileNotFoundError: python_x64 = r'C:\Python{}\python.exe'.format(ver) - try: - reg32_key = winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, - r'SOFTWARE\WOW6432Node\Python\PythonCore' - r'\{}-32\InstallPath'.format(dot_ver)) - python_x86 = winreg.QueryValueEx(reg32_key, 'ExecutablePath')[0] - except FileNotFoundError: - python_x86 = r'C:\Python{}-32\python.exe'.format(ver) - out_pyinstaller = os.path.join('dist', 'qutebrowser') - out_32 = os.path.join('dist', - 'qutebrowser-{}-x86'.format(qutebrowser.__version__)) out_64 = os.path.join('dist', 'qutebrowser-{}-x64'.format(qutebrowser.__version__)) artifacts = [] + from scripts.dev import gen_versioninfo utils.print_title("Updating VersionInfo file") gen_versioninfo.main() - utils.print_title("Running pyinstaller 32bit") - _maybe_remove(out_32) - call_tox('pyinstaller', '-r', python=python_x86) - shutil.move(out_pyinstaller, out_32) - patch_windows(out_32) - utils.print_title("Running pyinstaller 64bit") _maybe_remove(out_64) call_tox('pyinstaller', '-r', python=python_x64) @@ -271,39 +255,22 @@ def build_windows(): patch_windows(out_64) utils.print_title("Building installers") - subprocess.run(['makensis.exe', - '/DVERSION={}'.format(qutebrowser.__version__), - 'misc/qutebrowser.nsi'], check=True) subprocess.run(['makensis.exe', '/DX64', '/DVERSION={}'.format(qutebrowser.__version__), 'misc/qutebrowser.nsi'], check=True) - name_32 = 'qutebrowser-{}-win32.exe'.format(qutebrowser.__version__) name_64 = 'qutebrowser-{}-amd64.exe'.format(qutebrowser.__version__) artifacts += [ - (os.path.join('dist', name_32), - 'application/vnd.microsoft.portable-executable', - 'Windows 32bit installer'), (os.path.join('dist', name_64), 'application/vnd.microsoft.portable-executable', 'Windows 64bit installer'), ] - utils.print_title("Running 32bit smoke test") - smoke_test(os.path.join(out_32, 'qutebrowser.exe')) utils.print_title("Running 64bit smoke test") smoke_test(os.path.join(out_64, 'qutebrowser.exe')) - utils.print_title("Zipping 32bit standalone...") - name = 'qutebrowser-{}-windows-standalone-win32'.format( - qutebrowser.__version__) - shutil.make_archive(name, 'zip', 'dist', os.path.basename(out_32)) - artifacts.append(('{}.zip'.format(name), - 'application/zip', - 'Windows 32bit standalone')) - utils.print_title("Zipping 64bit standalone...") name = 'qutebrowser-{}-windows-standalone-amd64'.format( qutebrowser.__version__) diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index 20aa1c12d..18f5aa9ec 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -43,11 +43,6 @@ travis_retry() { return $result } -brew_install() { - brew update - brew install "$@" -} - pip_install() { travis_retry python3 -m pip install "$@" } @@ -62,7 +57,10 @@ check_pyqt() { python3 <=': not qtutils.version_check(version), - '<': qtutils.version_check(version), + '>=': not qtutils.version_check(version, compiled=False), + '<': qtutils.version_check(version, compiled=False), '!=': qtutils.version_check(version, exact=True, compiled=False), } return pytest.mark.skipif(do_skip[op], reason='Needs ' + tag) diff --git a/tests/end2end/data/issue4011.html b/tests/end2end/data/issue4011.html new file mode 100644 index 000000000..488193736 --- /dev/null +++ b/tests/end2end/data/issue4011.html @@ -0,0 +1,10 @@ + + + + + <img src="x" onerror="console.log('XSS')">foo + + + foo + + diff --git a/tests/end2end/data/misc/qutescheme_csrf.html b/tests/end2end/data/misc/qutescheme_csrf.html new file mode 100644 index 000000000..66c8fe240 --- /dev/null +++ b/tests/end2end/data/misc/qutescheme_csrf.html @@ -0,0 +1,20 @@ + + + + + CSRF issues with qute://settings + + + +

+ + Via link + Via redirect + + diff --git a/tests/end2end/data/userscripts/hello_if_count b/tests/end2end/data/userscripts/hello_if_count new file mode 100755 index 000000000..b9f07b86f --- /dev/null +++ b/tests/end2end/data/userscripts/hello_if_count @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ "$QUTE_COUNT" -eq 5 ]; then + + echo "message-info 'Count is five!'" >> "$QUTE_FIFO" + +elif [ -z "$QUTE_COUNT" ]; then + + echo "message-info 'No count!'" >> "$QUTE_FIFO" + +fi diff --git a/tests/end2end/features/backforward.feature b/tests/end2end/features/backforward.feature index 413ee9d95..4de785517 100644 --- a/tests/end2end/features/backforward.feature +++ b/tests/end2end/features/backforward.feature @@ -3,6 +3,7 @@ Feature: Going back and forward. Testing the :back/:forward commands. + @skip # Too flaky Scenario: Going back/forward Given I open data/backforward/1.txt When I open data/backforward/2.txt @@ -74,6 +75,7 @@ Feature: Going back and forward. url: http://localhost:*/data/backforward/1.txt - url: http://localhost:*/data/backforward/2.txt + @flaky Scenario: Going back with count. Given I open data/backforward/1.txt When I open data/backforward/2.txt diff --git a/tests/end2end/features/caret.feature b/tests/end2end/features/caret.feature index 268828edd..6d2245a89 100644 --- a/tests/end2end/features/caret.feature +++ b/tests/end2end/features/caret.feature @@ -321,6 +321,7 @@ Feature: Caret mode - data/caret.html - data/hello.txt (active) + @flaky Scenario: :follow-selected with link tabbing (without JS) When I set content.javascript.enabled to false And I run :leave-mode @@ -329,6 +330,7 @@ Feature: Caret mode And I run :follow-selected Then data/hello.txt should be loaded + @flaky Scenario: :follow-selected with link tabbing (with JS) When I set content.javascript.enabled to true And I run :leave-mode @@ -337,6 +339,7 @@ Feature: Caret mode And I run :follow-selected Then data/hello.txt should be loaded + @flaky Scenario: :follow-selected with link tabbing in a tab (without JS) When I set content.javascript.enabled to false And I run :leave-mode @@ -345,6 +348,7 @@ Feature: Caret mode And I run :follow-selected --tab Then data/hello.txt should be loaded + @flaky Scenario: :follow-selected with link tabbing in a tab (with JS) When I set content.javascript.enabled to true And I run :leave-mode diff --git a/tests/end2end/features/downloads.feature b/tests/end2end/features/downloads.feature index 7aa92abe0..e62c7a030 100644 --- a/tests/end2end/features/downloads.feature +++ b/tests/end2end/features/downloads.feature @@ -211,6 +211,7 @@ Feature: Downloading things from a website. # works e.g. on a connection loss, which we can't test automatically. Then "Retrying downloads is unsupported *" should not be logged + @flaky Scenario: Retrying with count When I run :download http://localhost:(port)/data/downloads/download.bin And I run :download http://localhost:(port)/does-not-exist diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 33535856c..ada688903 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -86,6 +86,7 @@ Feature: Opening external editors When I run :edit-url -t -b Then the error "Only one of -t/-b/-w can be given!" should be shown + @flaky Scenario: Editing a URL with invalid URL When I set url.auto_search to never And I open data/hello.txt diff --git a/tests/end2end/features/hints.feature b/tests/end2end/features/hints.feature index e33b16f68..3a0cb1da0 100644 --- a/tests/end2end/features/hints.feature +++ b/tests/end2end/features/hints.feature @@ -165,13 +165,13 @@ Feature: Using hints And I hint with args "all run message-info {hint-url}" and follow a Then the message "http://localhost:(port)/data/hello.txt" should be shown - @qt!=5.11.0 + @qt<5.11 Scenario: Clicking an invalid link When I open data/invalid_link.html And I hint with args "all" and follow a Then the error "Invalid link clicked - *" should be shown - @qt!=5.11.0 + @qt<5.11 Scenario: Clicking an invalid link opening in a new tab When I open data/invalid_link.html And I hint with args "all tab" and follow a diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index 2e2e1712a..0706fde17 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -96,19 +96,36 @@ Feature: Page history http://localhost:(port)/data/hints/html/simple.html Simple link http://localhost:(port)/data/hello.txt + @flaky Scenario: Listing history When I open data/numbers/3.txt And I open data/numbers/4.txt And I open qute://history + And I wait 2s Then the page should contain the plaintext "3.txt" Then the page should contain the plaintext "4.txt" # Hangs a lot on AppVeyor - @posix + @posix @flaky Scenario: Listing history with qute:history redirect When I open data/numbers/3.txt And I open data/numbers/4.txt And I open qute:history without waiting And I wait until qute://history is loaded + And I wait 2s Then the page should contain the plaintext "3.txt" Then the page should contain the plaintext "4.txt" + + Scenario: XSS in :history + When I open data/issue4011.html + And I open qute://history + Then the javascript message "XSS" should not be logged + + @skip # Too flaky + Scenario: Escaping of URLs in :history + When I open query?one=1&two=2 + And I open qute://history + And I wait 2s # JS loads the history async + And I hint with args "links normal" and follow a + And I wait until query?one=1&two=2 is loaded + Then the query parameter two should be set to 2 diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature index 8c4348e5f..72318ba15 100644 --- a/tests/end2end/features/javascript.feature +++ b/tests/end2end/features/javascript.feature @@ -8,7 +8,7 @@ Feature: Javascript stuff When I open data/javascript/consolelog.html Then the javascript message "console.log works!" should be logged - @flaky + @skip # Too flaky Scenario: Opening/Closing a window via JS When I open data/javascript/window_open.html And I run :tab-only @@ -21,7 +21,7 @@ Feature: Javascript stuff And the following tabs should be open: - data/javascript/window_open.html (active) - @qtwebkit_skip @flaky + @skip # Too flaky Scenario: Opening/closing a modal window via JS When I open data/javascript/window_open.html And I run :tab-only @@ -131,6 +131,7 @@ Feature: Javascript stuff And I run :tab-next Then the window sizes should be the same + @flaky Scenario: Checking visible/invisible window size with vertical tabbar When I run :tab-only And I set tabs.position to left diff --git a/tests/end2end/features/keyinput.feature b/tests/end2end/features/keyinput.feature index f681012f1..5456b6739 100644 --- a/tests/end2end/features/keyinput.feature +++ b/tests/end2end/features/keyinput.feature @@ -18,6 +18,7 @@ Feature: Keyboard input # input.forward_unbound_keys + @qt<5.11.1 Scenario: Forwarding all keys When I open data/keyinput/log.html And I set input.forward_unbound_keys to all @@ -30,6 +31,7 @@ Feature: Keyboard input And the javascript message "key press: 112" should be logged And the javascript message "key release: 112" should be logged + @qt<5.11.1 Scenario: Forwarding special keys When I open data/keyinput/log.html And I set input.forward_unbound_keys to auto diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index f3b5ac47a..5f0035b8b 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -533,14 +533,14 @@ Feature: Various utility commands. Then "Renderer process crashed" should be logged And "* 'Error loading chrome://crash/'" should be logged - @qtwebkit_skip @no_invalid_lines @qt>=5.9 + @qtwebkit_skip @no_invalid_lines @qt>=5.9 @flaky Scenario: Renderer kill (5.9) When I run :open -t chrome://kill Then "Renderer process was killed" should be logged And "* 'Error loading chrome://kill/'" should be logged # https://github.com/qutebrowser/qutebrowser/issues/2290 - @qtwebkit_skip @no_invalid_lines + @qtwebkit_skip @no_invalid_lines @flaky Scenario: Navigating to URL after renderer process is gone When I run :tab-only And I open data/numbers/1.txt diff --git a/tests/end2end/features/open.feature b/tests/end2end/features/open.feature index ebae3cefb..3aec41786 100644 --- a/tests/end2end/features/open.feature +++ b/tests/end2end/features/open.feature @@ -41,6 +41,7 @@ Feature: Opening pages And I run :open 3 Then data/numbers/3.txt should be loaded + @flaky Scenario: Opening in a new tab Given I open about:blank When I run :tab-only diff --git a/tests/end2end/features/qutescheme.feature b/tests/end2end/features/qutescheme.feature index 1abaadd87..150f74988 100644 --- a/tests/end2end/features/qutescheme.feature +++ b/tests/end2end/features/qutescheme.feature @@ -98,8 +98,8 @@ Feature: Special qute:// pages # qute://settings - # Sometimes, an unrelated value gets set - @flaky + # Sometimes, an unrelated value gets set, which also breaks other tests + @skip Scenario: Focusing input fields in qute://settings and entering valid value When I set search.ignore_case to never And I open qute://settings @@ -116,7 +116,8 @@ Feature: Special qute:// pages Then the option search.ignore_case should be set to always # Sometimes, an unrelated value gets set - @flaky + # Too flaky... + @skip Scenario: Focusing input fields in qute://settings and entering invalid value When I open qute://settings # scroll to the right - the table does not fit in the default screen @@ -130,6 +131,63 @@ Feature: Special qute:// pages And I press the key "" Then "Invalid value 'foo' *" should be logged + @qtwebkit_skip + Scenario: qute://settings CSRF via img (webengine) + When I open data/misc/qutescheme_csrf.html + And I run :click-element id via-img + Then "Blocking malicious request from http://localhost:*/data/misc/qutescheme_csrf.html to qute://settings/set?*" should be logged + + @qtwebkit_skip + Scenario: qute://settings CSRF via link (webengine) + When I open data/misc/qutescheme_csrf.html + And I run :click-element id via-link + Then "Blocking malicious request from qute://settings/set?* to qute://settings/set?*" should be logged + + @qtwebkit_skip + Scenario: qute://settings CSRF via redirect (webengine) + When I open data/misc/qutescheme_csrf.html + And I run :click-element id via-redirect + Then "Blocking malicious request from qute://settings/set?* to qute://settings/set?*" should be logged + + @qtwebkit_skip + Scenario: qute://settings CSRF via form (webengine) + When I open data/misc/qutescheme_csrf.html + And I run :click-element id via-form + Then "Blocking malicious request from qute://settings/set?* to qute://settings/set?*" should be logged + + @qtwebkit_skip + Scenario: qute://settings CSRF token (webengine) + When I open qute://settings + And I run :jseval const xhr = new XMLHttpRequest(); xhr.open("GET", "qute://settings/set"); xhr.send() + Then "Error while handling qute://* URL" should be logged + And the error "Invalid CSRF token for qute://settings!" should be shown + + @qtwebengine_skip + Scenario: qute://settings CSRF via img (webkit) + When I open data/misc/qutescheme_csrf.html + And I run :click-element id via-img + Then "Blocking malicious request from http://localhost:*/data/misc/qutescheme_csrf.html to qute://settings/set?*" should be logged + + @qtwebengine_skip + Scenario: qute://settings CSRF via link (webkit) + When I open data/misc/qutescheme_csrf.html + And I run :click-element id via-link + Then "Blocking malicious request from http://localhost:*/data/misc/qutescheme_csrf.html to qute://settings/set?*" should be logged + And "Error while loading qute://settings/set?*: Invalid qute://settings request" should be logged + + @qtwebengine_skip + Scenario: qute://settings CSRF via redirect (webkit) + When I open data/misc/qutescheme_csrf.html + And I run :click-element id via-redirect + Then "Blocking malicious request from http://localhost:*/data/misc/qutescheme_csrf.html to qute://settings/set?*" should be logged + And "Error while loading qute://settings/set?*: Invalid qute://settings request" should be logged + + @qtwebengine_skip + Scenario: qute://settings CSRF via form (webkit) + When I open data/misc/qutescheme_csrf.html + And I run :click-element id via-form + Then "Error while loading qute://settings/set?*: Unsupported request type" should be logged + # pdfjs support @qtwebengine_skip: pdfjs is not implemented yet diff --git a/tests/end2end/features/scroll.feature b/tests/end2end/features/scroll.feature index 83eef16ab..09c4cfa51 100644 --- a/tests/end2end/features/scroll.feature +++ b/tests/end2end/features/scroll.feature @@ -145,6 +145,7 @@ Feature: Scrolling And I wait until the scroll position changed to 0/0 Then the page should not be scrolled + @skip # Too flaky Scenario: Scrolling down with a very big count When I run :scroll down with count 99999999999 # Make sure it doesn't hang @@ -162,6 +163,7 @@ Feature: Scrolling When I run :scroll-to-perc 100 Then the page should be scrolled vertically + @flaky Scenario: Scrolling to bottom and to top with :scroll-to-perc When I run :scroll-to-perc 100 And I wait until the scroll position changed @@ -173,10 +175,12 @@ Feature: Scrolling When I run :scroll-to-perc 50 Then the page should be scrolled vertically + @flaky Scenario: Scrolling to middle with :scroll-to-perc (float) When I run :scroll-to-perc 50.5 Then the page should be scrolled vertically + @flaky Scenario: Scrolling to middle and to top with :scroll-to-perc When I run :scroll-to-perc 50 And I wait until the scroll position changed @@ -231,7 +235,7 @@ Feature: Scrolling Scenario: :scroll-to-perc with count and argument When I run :scroll-to-perc 0 with count 50 Then the page should be scrolled vertically - + # https://github.com/qutebrowser/qutebrowser/issues/1821 @issue3572 Scenario: :scroll-to-perc without doctype @@ -249,6 +253,7 @@ Feature: Scrolling When I run :scroll-page 0 1.5 Then the page should be scrolled vertically + @flaky Scenario: Scrolling down and up with :scroll-page When I run :scroll-page 0 1 And I wait until the scroll position changed diff --git a/tests/end2end/features/search.feature b/tests/end2end/features/search.feature index 568831c0d..f6c12e98d 100644 --- a/tests/end2end/features/search.feature +++ b/tests/end2end/features/search.feature @@ -214,12 +214,14 @@ Feature: Searching on a page # TODO: wrapping message without scrolling ## follow searched links + @flaky Scenario: Follow a searched link When I run :search follow And I wait for "search found follow" in the log And I run :follow-selected Then data/hello.txt should be loaded + @flaky Scenario: Follow a searched link in a new tab When I run :window-only And I run :search follow @@ -260,7 +262,8 @@ Feature: Searching on a page - data/search.html - data/hello.txt (active) - @qtwebkit_skip: Not supported in qtwebkit + # Too flaky + @qtwebkit_skip: Not supported in qtwebkit @skip Scenario: Follow a searched link in an iframe When I open data/iframe_search.html And I run :tab-only @@ -269,7 +272,7 @@ Feature: Searching on a page And I run :follow-selected Then "navigation request: url http://localhost:*/data/hello.txt, type Type.link_clicked, is_main_frame False" should be logged - @qtwebkit_skip: Not supported in qtwebkit + @qtwebkit_skip: Not supported in qtwebkit @flaky Scenario: Follow a tabbed searched link in an iframe When I open data/iframe_search.html And I run :tab-only diff --git a/tests/end2end/features/sessions.feature b/tests/end2end/features/sessions.feature index cb54e34cf..626a88ba8 100644 --- a/tests/end2end/features/sessions.feature +++ b/tests/end2end/features/sessions.feature @@ -228,7 +228,7 @@ Feature: Saving and loading sessions url: http://localhost:*/data/hello.txt # Seems like that bug is fixed upstream in QtWebEngine - @qtwebkit_skip + @qtwebkit_skip @flaky Scenario: Saving a session with a page using history.replaceState() and navigating away When I open data/sessions/history_replace_state.html without waiting And I wait for "* Called history.replaceState" in the log @@ -363,7 +363,7 @@ Feature: Saving and loading sessions And I replace "about:blank" by "http://localhost:(port)/data/numbers/2.txt" in the "loaded_session" session file And I run :session-load loaded_session Then data/numbers/2.txt should be loaded - + @qtwebengine_flaky Scenario: Loading and deleting a session When I open about:blank diff --git a/tests/end2end/features/spawn.feature b/tests/end2end/features/spawn.feature index 2a1ea0039..87ffb53e0 100644 --- a/tests/end2end/features/spawn.feature +++ b/tests/end2end/features/spawn.feature @@ -47,6 +47,17 @@ Feature: :spawn - data/hello.txt - data/hello.txt (active) + @posix + Scenario: Running :spawn with userscript and count + When I run :spawn -u (testdata)/userscripts/hello_if_count with count 5 + Then the message "Count is five!" should be shown + + @posix + Scenario: Running :spawn with userscript and no count + When I run :spawn -u (testdata)/userscripts/hello_if_count + Then the message "No count!" should be shown + + @windows Scenario: Running :spawn with userscript on Windows When I open data/hello.txt diff --git a/tests/end2end/features/tabs.feature b/tests/end2end/features/tabs.feature index 5f5ae5d29..3f1be405a 100644 --- a/tests/end2end/features/tabs.feature +++ b/tests/end2end/features/tabs.feature @@ -915,6 +915,7 @@ Feature: Tab management When I run :buffer invalid title Then the error "No matching tab for: invalid title" should be shown + @flaky Scenario: :buffer with matching title and two windows When I open data/title.html And I open data/search.html in a new tab diff --git a/tests/end2end/features/test_history_bdd.py b/tests/end2end/features/test_history_bdd.py index 6efa08330..4d477d832 100644 --- a/tests/end2end/features/test_history_bdd.py +++ b/tests/end2end/features/test_history_bdd.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . +import json import logging import re @@ -34,6 +35,19 @@ def turn_on_sql_history(quteproc): quteproc.wait_for_load_finished_url('qute://pyeval') +@bdd.then(bdd.parsers.parse("the query parameter {name} should be set to " + "{value}")) +def check_query(quteproc, name, value): + """Check if a given query is set correctly. + + This assumes we're on the server query page. + """ + content = quteproc.get_content() + data = json.loads(content) + print(data) + assert data[name] == value + + @bdd.then(bdd.parsers.parse("the history should contain:\n{expected}")) def check_history(quteproc, server, tmpdir, expected): path = tmpdir / 'history' diff --git a/tests/end2end/features/test_prompts_bdd.py b/tests/end2end/features/test_prompts_bdd.py index 12d9cbeec..0d74700b4 100644 --- a/tests/end2end/features/test_prompts_bdd.py +++ b/tests/end2end/features/test_prompts_bdd.py @@ -17,9 +17,13 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . +import time + import pytest_bdd as bdd bdd.scenarios('prompts.feature') +from qutebrowser.utils import qtutils + @bdd.when("I load an SSL page") def load_ssl_page(quteproc, ssl_server): @@ -46,14 +50,21 @@ def no_prompt_shown(quteproc): @bdd.then("a SSL error page should be shown") def ssl_error_page(request, quteproc): - if not request.config.webengine: - line = quteproc.wait_for(message='Error while loading *: SSL ' - 'handshake failed') - line.expected = True - quteproc.wait_for(message="Changing title for idx * to 'Error " - "loading page: *'") - content = quteproc.get_content().strip() - assert "Unable to load page" in content + if request.config.webengine and qtutils.version_check('5.9'): + quteproc.wait_for(message="Certificate error: *") + time.sleep(0.5) # Wait for error page to appear + content = quteproc.get_content().strip() + assert ("ERR_INSECURE_RESPONSE" in content or # Qt <= 5.10 + "ERR_CERT_AUTHORITY_INVALID" in content) # Qt 5.11 + else: + if not request.config.webengine: + line = quteproc.wait_for(message='Error while loading *: SSL ' + 'handshake failed') + line.expected = True + quteproc.wait_for(message="Changing title for idx * to 'Error " + "loading page: *'") + content = quteproc.get_content().strip() + assert "Unable to load page" in content class AbstractCertificateErrorWrapper: diff --git a/tests/end2end/features/urlmarks.feature b/tests/end2end/features/urlmarks.feature index 836af5c4f..ec38116c3 100644 --- a/tests/end2end/features/urlmarks.feature +++ b/tests/end2end/features/urlmarks.feature @@ -116,6 +116,7 @@ Feature: quickmarks and bookmarks When I run :quickmark-add http://localhost:(port)/data/numbers/9.txt nine Then the quickmark file should contain "nine http://localhost:*/data/numbers/9.txt" + @flaky Scenario: Saving a quickmark (:quickmark-save) When I open data/numbers/10.txt And I run :quickmark-save diff --git a/tests/end2end/features/utilcmds.feature b/tests/end2end/features/utilcmds.feature index 5ccbac9b3..fac335813 100644 --- a/tests/end2end/features/utilcmds.feature +++ b/tests/end2end/features/utilcmds.feature @@ -41,6 +41,15 @@ Feature: Miscellaneous utility commands exposed to the user. # If we have an error, the test will fail Then no crash should happen + Scenario: :repeat with count + When I run :repeat 3 message-info "repeat-test 3" with count 2 + Then the message "repeat-test 3" should be shown + And the message "repeat-test 3" should be shown + And the message "repeat-test 3" should be shown + And the message "repeat-test 3" should be shown + And the message "repeat-test 3" should be shown + And the message "repeat-test 3" should be shown + ## :run-with-count Scenario: :run-with-count diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index 4101e6142..e0ba4e00d 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -104,6 +104,15 @@ def is_ignored_lowlevel_message(message): # Qt 5.11 # DevTools listening on ws://127.0.0.1:37945/devtools/browser/... 'DevTools listening on *', + # /home/travis/build/qutebrowser/qutebrowser/.tox/py36-pyqt511-cov/lib/ + # python3.6/site-packages/PyQt5/Qt/libexec/QtWebEngineProcess: + # /lib/x86_64-linux-gnu/libdbus-1.so.3: no version information + # available (required by /home/travis/build/qutebrowser/qutebrowser/ + # .tox/py36-pyqt511-cov/lib/python3.6/site-packages/PyQt5/Qt/libexec/ + # ../lib/libQt5WebEngineCore.so.5) + '*/QtWebEngineProcess: /lib/x86_64-linux-gnu/libdbus-1.so.3: no ' + 'version information available (required by ' + '*/libQt5WebEngineCore.so.5)', ] return any(testutils.pattern_match(pattern=pattern, value=message) for pattern in ignored_messages) @@ -204,6 +213,10 @@ def is_ignored_chromium_message(line): # [30412:30412:0323/074933.387250:ERROR:node_channel.cc(899)] Dropping # message on closed channel. 'Dropping message on closed channel.', + # [2204:1408:0703/113804.788:ERROR: + # gpu_process_transport_factory.cc(1019)] Lost UI shared context. + 'Lost UI shared context.', + ] return any(testutils.pattern_match(pattern=pattern, value=message) for pattern in ignored_messages) diff --git a/tests/end2end/fixtures/webserver_sub.py b/tests/end2end/fixtures/webserver_sub.py index 7d9af2ee3..15cd0becc 100644 --- a/tests/end2end/fixtures/webserver_sub.py +++ b/tests/end2end/fixtures/webserver_sub.py @@ -261,6 +261,11 @@ def response_headers(): return response +@app.route('/query') +def query(): + return flask.jsonify(flask.request.args) + + @app.route('/user-agent') def view_user_agent(): """Return User-Agent.""" diff --git a/tests/end2end/test_hints_html.py b/tests/end2end/test_hints_html.py index abc94d70d..9dee23c53 100644 --- a/tests/end2end/test_hints_html.py +++ b/tests/end2end/test_hints_html.py @@ -117,6 +117,7 @@ def test_hints(test_name, zoom_text_only, zoom_level, find_implementation, quteproc.set_setting('hints.find_implementation', 'javascript') +@pytest.mark.skip # Too flaky def test_word_hints_issue1393(quteproc, tmpdir): dict_file = tmpdir / 'dict' dict_file.write(textwrap.dedent(""" diff --git a/tests/end2end/test_invocations.py b/tests/end2end/test_invocations.py index fbab681a4..341088db8 100644 --- a/tests/end2end/test_invocations.py +++ b/tests/end2end/test_invocations.py @@ -368,8 +368,11 @@ def test_qute_settings_persistence(short_tmpdir, request, quteproc_new): """Make sure settings from qute://settings are persistent.""" args = _base_args(request.config) + ['--basedir', str(short_tmpdir)] quteproc_new.start(args) - quteproc_new.open_path( - 'qute://settings/set?option=search.ignore_case&value=always') + quteproc_new.open_path('qute://settings/') + quteproc_new.send_cmd(':jseval --world main ' + 'cset("search.ignore_case", "always")') + quteproc_new.wait_for(message='No output or error') + assert quteproc_new.get_setting('search.ignore_case') == 'always' quteproc_new.send_cmd(':quit') diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 84e5b0125..c7f32edf9 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -459,6 +459,12 @@ class BookmarkManagerStub(UrlMarkManagerStub): pass +class SearchenginesManagerStub(UrlMarkManagerStub): + + """Stub for the bookmark-manager object.""" + + pass + class QuickmarkManagerStub(UrlMarkManagerStub): diff --git a/tests/unit/browser/test_adblock.py b/tests/unit/browser/test_adblock.py index 8ab3b930d..e4e7a0eda 100644 --- a/tests/unit/browser/test_adblock.py +++ b/tests/unit/browser/test_adblock.py @@ -259,6 +259,33 @@ def test_parsing_multiple_hosts_on_line(config_stub, basedir, download_stub, assert_urls(host_blocker, whitelisted=[]) +@pytest.mark.parametrize('ip, host', [ + ('127.0.0.1', 'localhost'), + ('27.0.0.1', 'localhost.localdomain'), + ('27.0.0.1', 'local'), + ('55.255.255.255', 'broadcasthost'), + (':1', 'localhost'), + (':1', 'ip6-localhost'), + (':1', 'ip6-loopback'), + ('e80::1%lo0', 'localhost'), + ('f00::0', 'ip6-localnet'), + ('f00::0', 'ip6-mcastprefix'), + ('f02::1', 'ip6-allnodes'), + ('f02::2', 'ip6-allrouters'), + ('ff02::3', 'ip6-allhosts'), + ('.0.0.0', '0.0.0.0'), + ('127.0.1.1', 'myhostname'), + ('127.0.0.53', 'myhostname'), +]) +def test_whitelisted_lines(config_stub, basedir, download_stub, data_tmpdir, + tmpdir, win_registry, caplog, ip, host): + """Make sure we don't block hosts we don't want to.""" + host_blocker = adblock.HostBlocker() + line = ('{} {}'.format(ip, host)).encode('ascii') + host_blocker._parse_line(line) + assert host not in host_blocker._blocked_hosts + + def test_failed_dl_update(config_stub, basedir, download_stub, data_tmpdir, tmpdir, win_registry, caplog): """One blocklist fails to download. diff --git a/tests/unit/browser/test_shared.py b/tests/unit/browser/test_shared.py index f24f7ad97..78302d8c1 100644 --- a/tests/unit/browser/test_shared.py +++ b/tests/unit/browser/test_shared.py @@ -19,6 +19,8 @@ import pytest +from PyQt5.QtCore import QUrl + from qutebrowser.browser import shared @@ -47,4 +49,4 @@ def test_custom_headers(config_stub, dnt, accept_language, custom_headers, headers.custom = custom_headers expected_items = sorted(expected.items()) - assert shared.custom_headers() == expected_items + assert shared.custom_headers(QUrl()) == expected_items diff --git a/tests/unit/browser/webengine/test_spell.py b/tests/unit/browser/webengine/test_spell.py index e8ce3cecc..14b343df5 100644 --- a/tests/unit/browser/webengine/test_spell.py +++ b/tests/unit/browser/webengine/test_spell.py @@ -17,14 +17,14 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""Tests for qutebrowser.browser.webengine.spell module.""" - import logging import os +import pytest from PyQt5.QtCore import QLibraryInfo + from qutebrowser.browser.webengine import spell -from qutebrowser.utils import usertypes +from qutebrowser.utils import usertypes, qtutils, standarddir def test_version(message_mock, caplog): @@ -38,10 +38,19 @@ def test_version(message_mock, caplog): assert msg.text == expected -def test_dictionary_dir(monkeypatch): - monkeypatch.setattr(QLibraryInfo, 'location', lambda _: 'datapath') - assert spell.dictionary_dir() == os.path.join('datapath', - 'qtwebengine_dictionaries') +@pytest.mark.parametrize('qt_version, old, subdir', [ + ('5.9', True, 'global_datapath'), + ('5.9', False, 'global_datapath'), + ('5.10', True, 'global_datapath'), + ('5.10', False, 'user_datapath'), +]) +def test_dictionary_dir(monkeypatch, qt_version, old, subdir): + monkeypatch.setattr(qtutils, 'qVersion', lambda: qt_version) + monkeypatch.setattr(QLibraryInfo, 'location', lambda _: 'global_datapath') + monkeypatch.setattr(standarddir, 'data', lambda: 'user_datapath') + + expected = os.path.join(subdir, 'qtwebengine_dictionaries') + assert spell.dictionary_dir(old=old) == expected def test_local_filename_dictionary_does_not_exist(monkeypatch): @@ -83,3 +92,60 @@ def test_local_filename_installed_malformed(tmpdir, monkeypatch, caplog): (tmpdir / lang_file).ensure() with caplog.at_level(logging.WARNING): assert spell.local_filename('en-US') == 'en-US-11-0' + + +class TestInit: + + ENV = 'QTWEBENGINE_DICTIONARIES_PATH' + + @pytest.fixture(autouse=True) + def remove_envvar(self, monkeypatch): + monkeypatch.delenv(self.ENV, raising=False) + + @pytest.fixture + def patch_new_qt(self, monkeypatch): + monkeypatch.setattr(spell.qtutils, 'version_check', + lambda _ver, compiled: True) + + @pytest.fixture + def dict_dir(self, data_tmpdir): + return data_tmpdir / 'qtwebengine_dictionaries' + + @pytest.fixture + def old_dict_dir(self, monkeypatch, tmpdir): + data_dir = tmpdir / 'old' + dict_dir = data_dir / 'qtwebengine_dictionaries' + (dict_dir / 'somedict').ensure() + monkeypatch.setattr(spell.QLibraryInfo, 'location', + lambda _arg: str(data_dir)) + return dict_dir + + def test_old_qt(self, monkeypatch): + monkeypatch.setattr(spell.qtutils, 'version_check', + lambda _ver, compiled: False) + spell.init() + assert self.ENV not in os.environ + + def test_new_qt(self, dict_dir, patch_new_qt): + spell.init() + assert os.environ[self.ENV] == str(dict_dir) + + def test_moving(self, old_dict_dir, dict_dir, patch_new_qt): + spell.init() + assert (dict_dir / 'somedict').exists() + + def test_moving_oserror(self, mocker, caplog, + old_dict_dir, dict_dir, patch_new_qt): + mocker.patch('shutil.copytree', side_effect=OSError) + + with caplog.at_level(logging.ERROR): + spell.init() + + record = caplog.records[0] + assert record.message == 'Failed to copy old dictionaries' + + def test_moving_existing_destdir(self, old_dict_dir, dict_dir, + patch_new_qt): + dict_dir.ensure(dir=True) + spell.init() + assert not (dict_dir / 'somedict').exists() diff --git a/tests/unit/browser/webkit/network/test_filescheme.py b/tests/unit/browser/webkit/network/test_filescheme.py index 5bdbb47cc..2654097ea 100644 --- a/tests/unit/browser/webkit/network/test_filescheme.py +++ b/tests/unit/browser/webkit/network/test_filescheme.py @@ -248,7 +248,7 @@ class TestFileSchemeHandler: def test_dir(self, tmpdir): url = QUrl.fromLocalFile(str(tmpdir)) req = QNetworkRequest(url) - reply = filescheme.handler(req) + reply = filescheme.handler(req, None, None) # The URL will always use /, even on Windows - so we force this here # too. tmpdir_path = str(tmpdir).replace(os.sep, '/') @@ -259,7 +259,7 @@ class TestFileSchemeHandler: filename.ensure() url = QUrl.fromLocalFile(str(filename)) req = QNetworkRequest(url) - reply = filescheme.handler(req) + reply = filescheme.handler(req, None, None) assert reply is None def test_unicode_encode_error(self, mocker): @@ -269,5 +269,5 @@ class TestFileSchemeHandler: err = UnicodeEncodeError('ascii', '', 0, 2, 'foo') mocker.patch('os.path.isdir', side_effect=err) - reply = filescheme.handler(req) + reply = filescheme.handler(req, None, None) assert reply is None diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index a8c7d9425..7cfc3d516 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -20,6 +20,8 @@ """Tests for completion models.""" import collections +import random +import string from datetime import datetime import pytest @@ -140,6 +142,24 @@ def configdata_stub(config_stub, monkeypatch, configdata_init): default=True, backends=[], raw_backends=None)), + ('completion.open_categories', configdata.Option( + name='completion.open_categories', + description='Which categories to show (in which order) in the :open completion.', + typ=configtypes.FlagList( + ), + default=["searchengines", "quickmarks", "bookmarks", "history"], + backends=[], + raw_backends=None)), + ('url.searchengines', configdata.Option( + name='url.searchengines', + description='searchengines list', + typ=configtypes.Dict( + keytype=configtypes.String(), + valtype=configtypes.String(), + ), + default={"DEFAULT": "https://duckduckgo.com/?q={}", "google": "https://google.com/?q={}"}, + backends=[], + raw_backends=None)), ])) config_stub._init_values() @@ -257,11 +277,100 @@ def test_help_completion(qtmodeltester, cmdutils_stub, key_config_stub, ('aliases', 'Aliases for commands.', None), ('bindings.commands', 'Default keybindings', None), ('bindings.default', 'Default keybindings', None), + ('completion.open_categories', 'Which categories to show (in which order) in the :open completion.', None), ('content.javascript.enabled', 'Enable/Disable JavaScript', None), - ] + ('url.searchengines', 'searchengines list', None), + ], }) +def test_open_categories(qtmodeltester, config_stub, web_history_populated, + quickmarks, bookmarks, info): + """Test that open_categories settings has the desired effect. + + Verify that: + - All categories are listed when they are defined in the completion.open_categories list + """ + config_stub.val.url.searchengines = {"DEFAULT": "https://duckduckgo.com/?q={}", + "google": "https://google.com/?q={}"} + config_stub.val.completion.open_categories = ["searchengines", "quickmarks", "bookmarks", "history"] + model = urlmodel.url(info=info) + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + _check_completions(model, { + "Search engines": [ + ('google', 'https://google.com/?q={}', None), + ], + "Quickmarks": [ + ('https://wiki.archlinux.org', 'aw', None), + ('https://wikipedia.org', 'wiki', None), + ('https://duckduckgo.com', 'ddg', None), + ], + "Bookmarks": [ + ('https://github.com', 'GitHub', None), + ('https://python.org', 'Welcome to Python.org', None), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), + ], + "History": [ + ('https://github.com', 'https://github.com', '2016-05-01'), + ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ], + }) + + +def test_open_categories_remove_all(qtmodeltester, config_stub, web_history_populated, + quickmarks, bookmarks, info): + """Test that removing an item (boookmarks) from the open_categories settings has the desired effect. + + Verify that: + - Only categories + """ + config_stub.val.url.searchengines = {"DEFAULT": "https://duckduckgo.com/?q={}", + "google": "https://google.com/?q={}"} + config_stub.val.completion.open_categories = [] + model = urlmodel.url(info=info) + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + _check_completions(model, { + }) + + +def test_open_categories_remove_one(qtmodeltester, config_stub, web_history_populated, + quickmarks, bookmarks, info): + """Test that removing an item (boookmarks) from the open_categories settings has the desired effect. + + Verify that: + - Only categories + """ + config_stub.val.url.searchengines = {"DEFAULT": "https://duckduckgo.com/?q={}", + "google": "https://google.com/?q={}"} + config_stub.val.completion.open_categories = ["searchengines", "quickmarks", "history"] + model = urlmodel.url(info=info) + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + _check_completions(model, { + "Search engines": [ + ('google', 'https://google.com/?q={}', None), + ], + "Quickmarks": [ + ('https://wiki.archlinux.org', 'aw', None), + ('https://wikipedia.org', 'wiki', None), + ('https://duckduckgo.com', 'ddg', None), + ], + "History": [ + ('https://github.com', 'https://github.com', '2016-05-01'), + ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ], + }) + def test_quickmark_completion(qtmodeltester, quickmarks): """Test the results of quickmark completion.""" model = miscmodels.quickmark() @@ -342,15 +451,53 @@ def url_args(fake_args): fake_args.debug_flags = [] -def test_url_completion(qtmodeltester, web_history_populated, +def test_url_completion(qtmodeltester, config_stub, web_history_populated, quickmarks, bookmarks, info): """Test the results of url completion. Verify that: - - quickmarks, bookmarks, and urls are included + - searchengines, quickmarks, bookmarks, and urls are included + - default search engine is not displayed - entries are sorted by access time - only the most recent entry is included for each url """ + config_stub.val.completion.open_categories = ["searchengines", "quickmarks", "bookmarks", "history"] + config_stub.val.url.searchengines = {"DEFAULT": "https://duckduckgo.com/?q={}", "google": "https://google.com/?q={}"} + model = urlmodel.url(info=info) + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + _check_completions(model, { + "Search engines": [ + ('google', 'https://google.com/?q={}', None), + ], + "Quickmarks": [ + ('https://wiki.archlinux.org', 'aw', None), + ('https://wikipedia.org', 'wiki', None), + ('https://duckduckgo.com', 'ddg', None), + ], + "Bookmarks": [ + ('https://github.com', 'GitHub', None), + ('https://python.org', 'Welcome to Python.org', None), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), + ], + "History": [ + ('https://github.com', 'https://github.com', '2016-05-01'), + ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ], + }) + +def test_search_only_default(qtmodeltester, config_stub, web_history_populated, + quickmarks, bookmarks, info): + """Test that Seardh engines is not shown when only default search engine is set in settings. + + Verify that: + - No Search engines categories is shown + """ + config_stub.val.completion.open_categories = ["searchengines", "quickmarks", "bookmarks", "history"] + config_stub.val.url.searchengines = {"DEFAULT": "https://duckduckgo.com/?q={}",} model = urlmodel.url(info=info) model.set_pattern('') qtmodeltester.data_display_may_return_none = True @@ -374,7 +521,6 @@ def test_url_completion(qtmodeltester, web_history_populated, ], }) - def test_url_completion_no_quickmarks(qtmodeltester, web_history_populated, quickmark_manager_stub, bookmarks, info): """Test that the quickmark category is gone with no quickmarks.""" @@ -517,9 +663,11 @@ def test_url_completion_zero_limit(config_stub, web_history, quickmarks, info, bookmarks): """Make sure there's no history if the limit was set to zero.""" config_stub.val.completion.web_history_max_items = 0 + config_stub.val.completion.open_categories = ["searchengines", "quickmarks", "bookmarks", "history"] + config_stub.val.url.searchengines = {"DEFAULT": "https://duckduckgo.com/?q={}", "google": "https://google.com/?q={}"} model = urlmodel.url(info=info) model.set_pattern('') - category = model.index(2, 0) # "History" normally + category = model.index(3, 0) # "History" normally assert model.data(category) is None @@ -593,6 +741,34 @@ def test_tab_completion_delete(qtmodeltester, fake_web_tab, app_stub, QUrl('https://duckduckgo.com')] +def test_tab_completion_not_sorted(qtmodeltester, fake_web_tab, app_stub, + win_registry, tabbed_browser_stubs): + """Ensure that the completion row order is the same as tab index order. + + Would be violated for more than 9 tabs if the completion was being + alphabetically sorted on the first column, or the others. + """ + expected = [] + for idx in range(1, 11): + url = "".join(random.sample(string.ascii_letters, 12)) + title = "".join(random.sample(string.ascii_letters, 12)) + expected.append(("0/{}".format(idx), url, title)) + + tabbed_browser_stubs[0].widget.tabs = [ + fake_web_tab(QUrl(tab[1]), tab[2], idx) + for idx, tab in enumerate(expected) + ] + model = miscmodels.buffer() + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + _check_completions(model, { + '0': expected, + '1': [], + }) + + def test_other_buffer_completion(qtmodeltester, fake_web_tab, app_stub, win_registry, tabbed_browser_stubs, info): tabbed_browser_stubs[0].widget.tabs = [ @@ -679,8 +855,11 @@ def test_setting_option_completion(qtmodeltester, config_stub, ('bindings.commands', 'Default keybindings', ( '{"normal": {"": "quit", "ZQ": "quit", ' '"I": "invalid", "d": "scroll down"}}')), + ('completion.open_categories', 'Which categories to show (in which order) in the :open completion.', '["searchengines", "quickmarks", "bookmarks", "history"]'), ('content.javascript.enabled', 'Enable/Disable JavaScript', 'true'), + ('url.searchengines', 'searchengines list', + '{"DEFAULT": "https://duckduckgo.com/?q={}", "google": "https://google.com/?q={}"}'), ] }) diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index e1ef7ef94..bf4f7c02d 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -642,8 +642,8 @@ class TestConfig: meth = getattr(conf, method) with pytest.raises(configexc.BackendError): with qtbot.assert_not_emitted(conf.changed): - meth('content.cookies.accept', 'all') - assert not conf._values['content.cookies.accept'] + meth('hints.find_implementation', 'javascript') + assert not conf._values['hints.find_implementation'] @pytest.mark.parametrize('method, value', [ ('set_obj', {}), diff --git a/tests/unit/config/test_configcommands.py b/tests/unit/config/test_configcommands.py index 343e7c8ca..77097d851 100644 --- a/tests/unit/config/test_configcommands.py +++ b/tests/unit/config/test_configcommands.py @@ -247,7 +247,7 @@ class TestCycle: commands.config_cycle(opt, '[foo]', '[bar]') assert config_stub.get(opt) == ['foo'] - def test_toggle(self, commands, config_stub): + def test_toggle(self, commands, config_stub, yaml_value): """Run ':config-cycle auto_save.session'. Should toggle the value. diff --git a/tests/unit/config/test_configinit.py b/tests/unit/config/test_configinit.py index cb32d3761..56f2a3c90 100644 --- a/tests/unit/config/test_configinit.py +++ b/tests/unit/config/test_configinit.py @@ -293,6 +293,8 @@ class TestEarlyInit: 'QT_XCB_FORCE_SOFTWARE_OPENGL', '1'), ('qt.force_software_rendering', 'qt-quick', 'QT_QUICK_BACKEND', 'software'), + ('qt.force_software_rendering', 'chromium', + 'QT_WEBENGINE_DISABLE_NOUVEAU_WORKAROUND', '1'), ('qt.force_platform', 'toaster', 'QT_QPA_PLATFORM', 'toaster'), ('qt.highdpi', True, 'QT_AUTO_SCREEN_SCALE_FACTOR', '1'), ('window.hide_decoration', True, diff --git a/tests/unit/keyinput/key_data.py b/tests/unit/keyinput/key_data.py index bf1ccdede..48b5c8c56 100644 --- a/tests/unit/keyinput/key_data.py +++ b/tests/unit/keyinput/key_data.py @@ -374,37 +374,36 @@ KEYS = [ Key('Dead_Hook', 'Hook', qtest=False), Key('Dead_Horn', 'Horn', qtest=False), - # Not in Qt 5.10, so data may be wrong! - Key('Dead_Stroke', qtest=False), - Key('Dead_Abovecomma', qtest=False), - Key('Dead_Abovereversedcomma', qtest=False), - Key('Dead_Doublegrave', qtest=False), - Key('Dead_Belowring', qtest=False), - Key('Dead_Belowmacron', qtest=False), - Key('Dead_Belowcircumflex', qtest=False), - Key('Dead_Belowtilde', qtest=False), - Key('Dead_Belowbreve', qtest=False), - Key('Dead_Belowdiaeresis', qtest=False), - Key('Dead_Invertedbreve', qtest=False), - Key('Dead_Belowcomma', qtest=False), - Key('Dead_Currency', qtest=False), - Key('Dead_a', qtest=False), - Key('Dead_A', qtest=False), - Key('Dead_e', qtest=False), - Key('Dead_E', qtest=False), - Key('Dead_i', qtest=False), - Key('Dead_I', qtest=False), - Key('Dead_o', qtest=False), - Key('Dead_O', qtest=False), - Key('Dead_u', qtest=False), - Key('Dead_U', qtest=False), - Key('Dead_Small_Schwa', qtest=False), - Key('Dead_Capital_Schwa', qtest=False), - Key('Dead_Greek', qtest=False), - Key('Dead_Lowline', qtest=False), - Key('Dead_Aboveverticalline', qtest=False), - Key('Dead_Belowverticalline', qtest=False), - Key('Dead_Longsolidusoverlay', qtest=False), + Key('Dead_Stroke', '̵', qtest=False), + Key('Dead_Abovecomma', '̓', qtest=False), + Key('Dead_Abovereversedcomma', '̔', qtest=False), + Key('Dead_Doublegrave', '̏', qtest=False), + Key('Dead_Belowring', '̥', qtest=False), + Key('Dead_Belowmacron', '̱', qtest=False), + Key('Dead_Belowcircumflex', '̭', qtest=False), + Key('Dead_Belowtilde', '̰', qtest=False), + Key('Dead_Belowbreve', '̮', qtest=False), + Key('Dead_Belowdiaeresis', '̤', qtest=False), + Key('Dead_Invertedbreve', '̑', qtest=False), + Key('Dead_Belowcomma', '̦', qtest=False), + Key('Dead_Currency', '¤', qtest=False), + Key('Dead_a', 'a', qtest=False), + Key('Dead_A', 'A', qtest=False), + Key('Dead_e', 'e', qtest=False), + Key('Dead_E', 'E', qtest=False), + Key('Dead_i', 'i', qtest=False), + Key('Dead_I', 'I', qtest=False), + Key('Dead_o', 'o', qtest=False), + Key('Dead_O', 'O', qtest=False), + Key('Dead_u', 'u', qtest=False), + Key('Dead_U', 'U', qtest=False), + Key('Dead_Small_Schwa', 'ə', qtest=False), + Key('Dead_Capital_Schwa', 'Ə', qtest=False), + Key('Dead_Greek', 'Greek', qtest=False), + Key('Dead_Lowline', '̲', qtest=False), + Key('Dead_Aboveverticalline', '̍', qtest=False), + Key('Dead_Belowverticalline', '\u0329', qtest=False), + Key('Dead_Longsolidusoverlay', '̸', qtest=False), ### multimedia/internet keys - ignored by default - see QKeyEvent c'tor Key('Back'), diff --git a/tests/unit/keyinput/test_keyutils.py b/tests/unit/keyinput/test_keyutils.py index dc8fe0a53..d6e7bce34 100644 --- a/tests/unit/keyinput/test_keyutils.py +++ b/tests/unit/keyinput/test_keyutils.py @@ -214,8 +214,10 @@ class TestKeySequence: @pytest.mark.parametrize('orig, normalized', [ ('', ''), ('', ''), - ('', ''), ('', ''), + ('', ''), + ('', ''), + ('', ''), ('', ''), ('', ''), ('', ''), diff --git a/tests/unit/misc/test_lineparser.py b/tests/unit/misc/test_lineparser.py index c3624cc1d..10575e676 100644 --- a/tests/unit/misc/test_lineparser.py +++ b/tests/unit/misc/test_lineparser.py @@ -37,21 +37,11 @@ class TestBaseLineParser: """Fixture providing a BaseLineParser.""" return lineparsermod.BaseLineParser(self.CONFDIR, self.FILENAME) - def test_prepare_save_existing(self, mocker, lineparser): - """Test if _prepare_save does what it's supposed to do.""" - os_mock = mocker.patch('qutebrowser.misc.lineparser.os') - os_mock.path.exists.return_value = True - - lineparser._prepare_save() - assert not os_mock.makedirs.called - def test_prepare_save_missing(self, mocker, lineparser): """Test if _prepare_save does what it's supposed to do.""" os_mock = mocker.patch('qutebrowser.misc.lineparser.os') - os_mock.path.exists.return_value = False - lineparser._prepare_save() - os_mock.makedirs.assert_called_with(self.CONFDIR, 0o755) + os_mock.makedirs.assert_called_with(self.CONFDIR, 0o755, exist_ok=True) def test_double_open(self, mocker, lineparser): """Test if _open refuses reentry.""" diff --git a/tests/unit/utils/test_qtutils.py b/tests/unit/utils/test_qtutils.py index 6de085b25..0825d1fb3 100644 --- a/tests/unit/utils/test_qtutils.py +++ b/tests/unit/utils/test_qtutils.py @@ -25,13 +25,6 @@ import os import os.path import unittest import unittest.mock -try: - # pylint: disable=no-name-in-module,useless-suppression - from test import test_file - # pylint: enable=no-name-in-module,useless-suppression -except ImportError: - # Debian patches Python to remove the tests... - test_file = None import pytest from PyQt5.QtCore import (QDataStream, QPoint, QUrl, QByteArray, QIODevice, @@ -40,6 +33,20 @@ from PyQt5.QtCore import (QDataStream, QPoint, QUrl, QByteArray, QIODevice, from qutebrowser.utils import qtutils, utils import overflow_test_cases +if utils.is_linux: + # Those are not run on macOS because that seems to cause a hang sometimes. + # On Windows, we don't run them either because of + # https://github.com/pytest-dev/pytest/issues/3650 + try: + # pylint: disable=no-name-in-module,useless-suppression + from test import test_file + # pylint: enable=no-name-in-module,useless-suppression + except ImportError: + # Debian patches Python to remove the tests... + test_file = None +else: + test_file = None + # pylint: disable=bad-continuation @pytest.mark.parametrize(['qversion', 'compiled', 'pyqt', 'version', 'exact', @@ -476,13 +483,11 @@ class TestSavefileOpen: assert data == b'foo\nbar\nbaz' -if test_file is not None and not utils.is_mac: +if test_file is not None: # If we were able to import Python's test_file module, we run some code # here which defines unittest TestCases to run the python tests over # PyQIODevice. - # Those are not run on macOS because that seems to cause a hang sometimes. - @pytest.fixture(scope='session', autouse=True) def clean_up_python_testfile(): """Clean up the python testfile after tests if tests didn't.""" @@ -570,7 +575,7 @@ class FailingQIODevice(QIODevice): self.setErrorString("Writing failed") return -1 - def read(self, _maxsize): + def read(self, _maxsize): # pylint: disable=useless-return """Simulate failed read.""" self.setErrorString("Reading failed") return None diff --git a/tests/unit/utils/test_standarddir.py b/tests/unit/utils/test_standarddir.py index a483ede64..b9777f670 100644 --- a/tests/unit/utils/test_standarddir.py +++ b/tests/unit/utils/test_standarddir.py @@ -183,10 +183,13 @@ class TestStandardDir: @pytest.mark.qt_log_ignore(r'^QStandardPaths: ') def test_linux_invalid_runtimedir(self, monkeypatch, tmpdir): """With invalid XDG_RUNTIME_DIR, fall back to TempLocation.""" + tmpdir_env = tmpdir / 'temp' + tmpdir_env.ensure(dir=True) monkeypatch.setenv('XDG_RUNTIME_DIR', str(tmpdir / 'does-not-exist')) - monkeypatch.setenv('TMPDIR', str(tmpdir / 'temp')) + monkeypatch.setenv('TMPDIR', tmpdir_env) + standarddir._init_dirs() - assert standarddir.runtime() == str(tmpdir / 'temp' / APPNAME) + assert standarddir.runtime() == str(tmpdir_env / APPNAME) @pytest.mark.fake_os('windows') def test_runtimedir_empty_tempdir(self, monkeypatch, tmpdir): diff --git a/tests/unit/utils/test_urlmatch.py b/tests/unit/utils/test_urlmatch.py index dcd703790..b77588469 100644 --- a/tests/unit/utils/test_urlmatch.py +++ b/tests/unit/utils/test_urlmatch.py @@ -84,6 +84,19 @@ from qutebrowser.utils import urlmatch # Additional tests ("http://[", "Invalid IPv6 URL"), + ("http://[fc2e:bb88::edac]:", "Invalid port: Port is empty"), + ("http://[fc2e::bb88::edac]", 'Invalid IPv6 address; source was "fc2e::bb88::edac"; host = ""'), + ("http://[fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac]", 'Invalid IPv6 address; source was "fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac"; host = ""'), + ("http://[fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac]", 'Invalid IPv6 address; source was "fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac"; host = ""'), + ("http://[127.0.0.1:fc2e::bb88:edac]", 'Invalid IPv6 address; source was "127.0.0.1:fc2e::bb88:edac'), + ("http://[]:20", "Pattern without host"), + ("http://[fc2e::bb88", "Invalid IPv6 URL"), + ("http://[[fc2e::bb88:edac]", """Expected ']' to match '[' in hostname; source was "[fc2e::bb88:edac"; host = """""), + pytest.param("http://[fc2e::bb88:edac]]", "Invalid IPv6 URL", marks=pytest.mark.xfail(reason="https://bugs.python.org/issue34360")), + ("http://[fc2e:bb88:edac]", 'Invalid IPv6 address; source was "fc2e:bb88:edac"; host = ""'), + ("http://[fc2e:bb88:edac::z]", 'Invalid IPv6 address; source was "fc2e:bb88:edac::z"; host = ""'), + ("http://[fc2e:bb88:edac::2]:2a2", "Invalid port: invalid literal for int() with base 10: '2a2'"), + ]) def test_invalid_patterns(pattern, error): with pytest.raises(urlmatch.ParseError, match=re.escape(error)): @@ -154,10 +167,16 @@ class TestMatchAllPagesForGivenScheme: @pytest.mark.parametrize('url, expected', [ ("http://google.com", True), + ("http://google.com:80", True), + ("http://google.com.", True), ("http://yahoo.com", True), ("http://google.com/foo", True), ("https://google.com", False), ("http://74.125.127.100/search", True), + ("http://[fc2e:0e35:bb88::edac]", True), + ("http://[fc2e:e35:bb88::edac]", True), + ("http://[fc2e:e35:bb88::127.0.0.1]", True), + ("http://[::1]/bar", True), ]) def test_urls(self, up, url, expected): assert up.matches(QUrl(url)) == expected @@ -238,6 +257,10 @@ class TestMatchIpAddresses: @pytest.mark.parametrize('pattern, host, match_subdomains', [ ("http://127.0.0.1/*", "127.0.0.1", False), ("http://*.0.0.1/*", "0.0.1", True), + ("http://[::1]/*", "::1", False), + ("http://[0::1]/*", "::1", False), + ("http://[::01]/*", "::1", False), + ("http://[0:0:0:0:20::1]/*", "::20:0:0:1", False), ]) def test_attrs(self, pattern, host, match_subdomains): up = urlmatch.UrlPattern(pattern) diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index ef2b6c8d4..550213386 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -410,11 +410,11 @@ class GotException(Exception): def excepthook(_exc, _val, _tb): - return + pass def excepthook_2(_exc, _val, _tb): - return + pass class TestDisabledExcepthook: @@ -462,7 +462,9 @@ class TestPreventExceptions: def test_raising(self, caplog): """Test with a raising function.""" with caplog.at_level(logging.ERROR, 'misc'): + # pylint: disable=assignment-from-no-return ret = self.func_raising() + # pylint: enable=assignment-from-no-return assert ret == 42 assert len(caplog.records) == 1 expected = 'Error in test_utils.TestPreventExceptions.func_raising' @@ -487,7 +489,9 @@ class TestPreventExceptions: def test_predicate_true(self, caplog): """Test with a True predicate.""" with caplog.at_level(logging.ERROR, 'misc'): + # pylint: disable=assignment-from-no-return ret = self.func_predicate_true() + # enable: disable=assignment-from-no-return assert ret == 42 assert len(caplog.records) == 1 diff --git a/tests/unit/utils/test_version.py b/tests/unit/utils/test_version.py index 683fba02e..2120a4b97 100644 --- a/tests/unit/utils/test_version.py +++ b/tests/unit/utils/test_version.py @@ -659,7 +659,7 @@ class TestModuleVersions: The aim of this test is to fail if that gets missing in some future version of sip. """ - import sip + from qutebrowser.qt import sip assert isinstance(sip.SIP_VERSION_STR, str) @@ -987,11 +987,12 @@ def test_pastebin_version(pbclient, message_mock, monkeypatch, qtbot): monkeypatch.setattr('qutebrowser.utils.utils.log_clipboard', True) version.pastebin_version(pbclient) - pbclient.success.emit("test") + pbclient.success.emit("https://www.example.com/\n") msg = message_mock.getmsg(usertypes.MessageLevel.info) - assert msg.text == "Version url test yanked to clipboard." - assert version.pastebin_url == "test" + expected_text = "Version url https://www.example.com/ yanked to clipboard." + assert msg.text == expected_text + assert version.pastebin_url == "https://www.example.com/" def test_pastebin_version_twice(pbclient, monkeypatch): @@ -1000,16 +1001,16 @@ def test_pastebin_version_twice(pbclient, monkeypatch): lambda: "dummy") version.pastebin_version(pbclient) - pbclient.success.emit("test") + pbclient.success.emit("https://www.example.com/\n") pbclient.url = None pbclient.data = None - version.pastebin_url = "test2" + version.pastebin_url = "https://www.example.org/" version.pastebin_version(pbclient) assert pbclient.url is None assert pbclient.data is None - assert version.pastebin_url == "test2" + assert version.pastebin_url == "https://www.example.org/" def test_pastebin_version_error(pbclient, caplog, message_mock, monkeypatch): diff --git a/tox.ini b/tox.ini index 4e155df57..02f3ae729 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py36-pyqt59-cov,misc,vulture,flake8,pylint,pyroma,check-manifest,eslint +envlist = py36-pyqt511-cov,misc,vulture,flake8,pylint,pyroma,check-manifest,eslint distshare = {toxworkdir} skipsdist = true @@ -13,14 +13,14 @@ skipsdist = true setenv = QT_QPA_PLATFORM_PLUGIN_PATH={envdir}/Lib/site-packages/PyQt5/plugins/platforms PYTEST_QT_API=pyqt5 - pyqt{,56,571,59,510}: LINK_PYQT_SKIP=true - pyqt{,56,571,59,510}: QUTE_BDD_WEBENGINE=true + pyqt{,56,571,59,510,511}: LINK_PYQT_SKIP=true + pyqt{,56,571,59,510,511}: QUTE_BDD_WEBENGINE=true cov: PYTEST_ADDOPTS=--cov --cov-report xml --cov-report=html --cov-report= passenv = PYTHON DISPLAY XAUTHORITY HOME USERNAME USER CI TRAVIS XDG_* QUTE_* DOCKER basepython = - py35: python3.5 - py36: python3.6 - py37: python3.7 + py35: {env:PYTHON:python3.5} + py36: {env:PYTHON:python3.6} + py37: {env:PYTHON:python3.7} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-tests.txt @@ -28,6 +28,7 @@ deps = pyqt571: PyQt5==5.7.1 pyqt59: PyQt5==5.9.2 pyqt510: PyQt5==5.10.1 + pyqt511: PyQt5==5.11.2 commands = {envpython} scripts/link_pyqt.py --tox {envdir} {envpython} -bb -m pytest {posargs:tests} @@ -59,7 +60,20 @@ commands = {envpython} -c "" usedevelop = true deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/misc/requirements/requirements-pyqt-old.txt + -r{toxinidir}/misc/requirements/requirements-pyqt.txt + +# Older PyQt for Python 3.5 +# 5.11.2: https://www.riverbankcomputing.com/pipermail/pyqt/2018-July/040511.html +# 5.10.1: https://github.com/qutebrowser/qutebrowser/issues/3662 +[testenv:mkvenv-pypi-old] +basepython = {env:PYTHON:python3.5} +envdir = {toxinidir}/.venv +commands = {envpython} -c "" +usedevelop = true +deps = + -r{toxinidir}/requirements.txt + PyQt5==5.10 + sip==4.19.8 [testenv:misc] ignore_errors = true