Merge branch 'master' into jay/pintab

.appveyor.yml

@@ -7,11 +7,14 @@ environment:
     PYTHONUNBUFFERED: 1
     matrix:
     - TESTENV: py34
+    - TESTENV: py36-pyqt58
+      PYTHON: C:\Python36\python.exe
     - TESTENV: unittests-frozen
     - TESTENV: pylint
 
 install:
   - C:\Python27\python -u scripts\dev\ci\appveyor_install.py
+  - set PATH=%PATH%;C:\Python36
 
 test_script:
   - C:\Python34\Scripts\tox -e %TESTENV%

.flake8

@@ -35,9 +35,9 @@ max-complexity = 12
 putty-auto-ignore = True
 putty-ignore =
     /# pylint: disable=invalid-name/ : +N801,N806
-    /# pylint: disable=wildcard-import/ : +F403
     /# pragma: no mccabe/ : +C901
     tests/*/test_*.py : +D100,D101,D401
+    tests/conftest.py : +F403
     tests/unit/browser/webkit/test_history.py : +N806
     tests/helpers/fixtures.py : +N806
     tests/unit/browser/webkit/http/test_content_disposition.py : +D400

.github/ISSUE_TEMPLATE.md

@@ -1,2 +1,2 @@
 <!-- If this is a bug report, please remember to mention your version info from
-the `qute:version` page or `qutebrowser --version` -->
+`:open qute:version` or `qutebrowser --version` -->

.pylintrc

@@ -38,7 +38,8 @@ disable=no-self-use,
         suppressed-message,
         too-many-return-statements,
         duplicate-code,
-        wrong-import-position
+        wrong-import-position,
+        no-else-return
 
 [BASIC]
 function-rgx=[a-z_][a-z0-9_]{2,50}$

CHANGELOG.asciidoc

@@ -21,18 +21,57 @@ Added
 ~~~~~
 
 - New `:clear-messages` command to clear shown messages.
+- New `ui -> keyhint-delay` setting to configure the delay until
+  the keyhint overlay pops up.
+- New `-s` option for `:open` to force a HTTPS scheme.
+- `:debug-log-filter` now accepts `none` as an argument to clear any log
+  filters.
+- New `--debug-flag` argument which replaces `--debug-exit` and
+  `--pdb-postmortem`.
+- New `tabs -> favicon-scale` option to scale up/down favicons.
 
 Changed
 ~~~~~~~
 
+- To prevent elaborate phishing attacks, the Punycode version is now shown in
+  addition to the decoded version for IDN domain names.
 - When using QtWebEngine, the underlying Chromium version is now shown in the
   version info.
+- Improved `qute:history` page with lazy loading
+- Messages are now hidden when clicked
+- Paths like `C:` are now treated as absolute paths on Windows for downloads,
+  and invalid paths are handled properly.
+- PAC on QtWebKit now supports SOCKS5 as type.
+- Comments in the config file are now before the individual
+  options instead of being before sections.
+- The HTTP cache is disabled with QtWebKit on Qt 5.7.1 and 5.8 now as it leads
+  to frequent crashes due to a Qt bug.
+- stdin is now closed immediately for processes spawned from qutebrowser
+- When ui -> message-timeout is set to 0, messages are now never cleared.
+- Middle/right-clicking the blank parts of the tab bar (when vertical) now
+  closes the current tab.
+- With Qt 5.9, `content -> cookies-store` can now be set on QtWebEngine without
+  a restart.
+- The adblocker now also blocks non-GET requests (e.g. POST)
+- `:follow-selected` now also works with QtWebEngine
 
 Fixed
 ~~~~~
 
 - Added a workaround for a black screen with QtWebEngine with some setups
-  (requires PyOpenGL to be installed)
+  (the workaround requires PyOpenGL to be installed, but it's optional)
+- Crash when trying to retry downloads with QtWebEngine
+- Crash when cloning page without history
+- Continuing a search after clearing it
+- Crash when downloading a download resulting in a HTTP error
+- Crash when pressing ctrl-c while a config error is shown
+- Crash when the key config isn't writable
+- Crash when unbinding an unbound key in the key config
+- Crash when using `:debug-log-filter` when `--filter` wasn't given on startup.
+- Crash with some invalid setting values
+- Crash when cloning a view-source tab with QtWebEngine
+- Various rare crashes
+- Various styling issues with the tabbar and a crash with qt5ct
 
 v0.10.1
 -------

CONTRIBUTING.asciidoc

@@ -42,6 +42,12 @@ be easy to solve]
 * https://github.com/qutebrowser/qutebrowser/labels/not%20code[Issues which
 require little/no coding]
 
+If you prefer C++ or Javascript to Python, see the relevant issues which involve
+work in those languages:
+
+* https://github.com/qutebrowser/qutebrowser/issues?utf8=%E2%9C%93&q=is%3Aopen%20is%3Aissue%20label%3Ac%2B%2B[C++] (mostly work on Qt, the library behind qutebrowser)
+* https://github.com/qutebrowser/qutebrowser/issues?q=is%3Aopen+is%3Aissue+label%3Ajavascript[JavaScript]
+
 There are also some things to do if you don't want to write code:
 
 * Help the community, e.g., on the mailinglist and the IRC channel.

FAQ.asciidoc

@@ -124,6 +124,34 @@ When using quickmark, you can give them all names, like
 `:open foodrecipes`, you will see a list of all the food recipe sites,
 without having to remember the exact website title or address.
 
+How do I use spell checking?::
+    Qutebrowser's support for spell checking is somewhat limited at the moment
+    (see https://github.com/qutebrowser/qutebrowser/issues/700[#700]), but it
+    can be done.
++
+For QtWebKit:
+
+. Install https://github.com/QupZilla/qtwebkit-plugins[qtwebkit-plugins].
+  . Note: with QtWebKit reloaded you may experience some issues. See
+    https://github.com/QupZilla/qtwebkit-plugins/issues/10[#10].
+. The dictionary to use is taken from the `DICTIONARY` environment variable.
+  The default is `en_US`. For example to use Dutch spell check set `DICTIONARY`
+  to `nl_NL`; you can't use multiple dictionaries or change them at runtime at
+  the moment.
+  (also see the README file for `qtwebkit-plugins`).
+. Remember to install the hunspell dictionaries if you don't have them already
+  (most distros should have packages for this).
+
++
+For QtWebEngine:
+
+. Not yet supported unfortunately :-( +
+  Adding it shouldn't be too hard though, since QtWebEngine 5.8 added an API for
+  this (see
+  https://github.com/qutebrowser/qutebrowser/issues/700#issuecomment-290780706[this
+  comment for a basic example]), so what are you waiting for and why aren't you
+  hacking qutebrowser yet?
+
 == Troubleshooting
 
 Configuration not saved after modifying config.::

INSTALL.asciidoc

@@ -135,12 +135,42 @@ If video or sound don't seem to work, try installing the gstreamer plugins:
 On Gentoo
 ---------
 
-qutebrowser is available in the main repository and can be installed with:
+A version of qutebrowser is available in the main repository and can be installed with:
 
 ----
 # emerge -av qutebrowser
 ----
 
+However it is suggested to install the Live version (-9999) to take advantage
+of the newest features introduced.
+
+First of all you need to edit your package.accept_keywords file to accept the live
+version:
+
+----
+# nano /etc/portage/package.accept_keywords
+----
+
+And add the following line to it:
+
+ =www-client/qutebrowser-9999 **
+
+Save the file and then install qutebrowser via
+
+----
+# emerge -av qutebrowser
+----
+
+Or rebuild your system if you already installed it.
+
+To update to the last Live version, remember to do
+
+----
+# emerge -uDNav @live-rebuild @world
+----
+
+To include qutebrowser among the updates.
+
 Make sure you have `python3_4` in your `PYTHON_TARGETS`
 (`/etc/portage/make.conf`) and rebuild your system (`emerge -uDNav @world`) if
 necessary.
@@ -192,19 +222,19 @@ On OpenBSD
 
 qutebrowser is in http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/www/qutebrowser/[OpenBSD ports].
 
-Manual install:
+Install the package:
+
+----
+# pkg_add qutebrowser
+----
+
+Or alternatively, use the ports system :
 
 ----
 # cd /usr/ports/www/qutebrowser
 # make install
 ----
 
-Or alternatively if you're using `-current` (or OpenBSD 6.1 once it's been released):
-
-----
-# pkg_add qutebrowser
-----
-
 On Windows
 ----------
 

README.asciidoc

@@ -24,6 +24,13 @@ on Python and PyQt5 and free software, licensed under the GPL.
 
 It was inspired by other browsers/addons like dwb and Vimperator/Pentadactyl.
 
+// QUTE_WEB_HIDE
+**qutebrowser is currently running a crowdfunding campaign for its new
+configuration system, allowing for per-domain settings and much more.**
+
+See the link:https://www.kickstarter.com/projects/the-compiler/qutebrowser-v10-with-per-domain-settings?ref=6zw7qz[Kickstarter campaign] for more information!
+// QUTE_WEB_HIDE_END
+
 Screenshots
 -----------
 
@@ -71,7 +78,8 @@ https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[mailinglist] at
 mailto:qutebrowser@lists.qutebrowser.org[].
 
 There's also a https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[announce-only mailinglist]
-at mailto:qutebrowser-announce@lists.qutebrowser.org[].
+at mailto:qutebrowser-announce@lists.qutebrowser.org[] (the announcements also
+get sent to the general qutebrowser@ list).
 
 Contributions / Bugs
 --------------------
@@ -152,38 +160,41 @@ Contributors, sorted by the number of commits in descending order:
 * Lamar Pavel
 * Marshall Lochbaum
 * Bruno Oliveira
+* Martin Tournoij
+* Imran Sobir
 * Alexander Cogneau
 * Felix Van der Jeugt
 * Daniel Karbach
-* Martin Tournoij
 * Kevin Velghe
 * Raphael Pierzina
 * Joel Torstensson
 * Patric Schmitz
 * Tarcisio Fedrizzi
 * Claude
+* Fritz Reichwald
 * Corentin Julé
 * meles5
 * Philipp Hansch
-* Imran Sobir
 * Panagiotis Ktistakis
 * Artur Shaik
 * Nathan Isom
 * Thorsten Wißmann
 * Austin Anderson
-* Fritz Reichwald
 * Jimmy
 * Niklas Haas
 * Maciej Wołczyk
-* Spreadyy
+* sandrosc
 * Alexey "Averrin" Nabrodov
+* pkill9
 * nanjekyejoannah
 * avk
 * ZDarian
 * Milan Svoboda
 * John ShaggyTwoDope Jenkins
+* Jay Kamat
 * Clayton Craft
 * Peter Vilim
+* Jacob Sword
 * knaggita
 * Oliver Caldwell
 * Julian Weigt
@@ -205,7 +216,6 @@ Contributors, sorted by the number of commits in descending order:
 * David Vogt
 * Claire Cavanaugh
 * rikn00
-* pkill9
 * kanikaa1234
 * haitaka
 * Nick Ginther
@@ -239,8 +249,10 @@ Contributors, sorted by the number of commits in descending order:
 * adam
 * Samir Benmendil
 * Regina Hug
+* Penaz
 * Mathias Fussenegger
 * Marcelo Santos
+* Marcel Schilling
 * Joel Bradshaw
 * Jean-Louis Fuchs
 * Franz Fellner
@@ -253,6 +265,7 @@ Contributors, sorted by the number of commits in descending order:
 * haxwithaxe
 * evan
 * dylan araps
+* caveman
 * addictedtoflames
 * Xitian9
 * Vasilij Schneidermann
@@ -261,19 +274,18 @@ Contributors, sorted by the number of commits in descending order:
 * Tobias Werth
 * Tim Harder
 * Thiago Barroso Perrotta
+* Steve Peak
 * Sorokin Alexei
 * Simon Désaulniers
 * Rok Mandeljc
 * Noah Huesser
 * Moez Bouhlel
 * Matthias Lisin
-* Marcel Schilling
 * Lazlow Carmichael
 * Kevin Wang
 * Ján Kobezda
 * Johannes Martinsson
 * Jean-Christophe Petkovich
-* Jay Kamat
 * Helen Sherwood-Taylor
 * HalosGhost
 * Gregor Pohl
@@ -281,9 +293,11 @@ Contributors, sorted by the number of commits in descending order:
 * Dietrich Daroch
 * Derek Sivers
 * Daniel Lu
+* Daniel Jakots
 * Arseniy Seroka
 * Andy Balaam
 * Andreas Fischer
+* Amos Bird
 * Akselmo
 // QUTE_AUTHORS_END
 

codecov.yml

@@ -1,9 +1,7 @@
+coverage:
   status:
-  project:
+    project: off
-    enabled: no
+    patch: off
-  patch:
+    changes: off
-    enabled: no
-  changes:
-    enabled: no
 
 comment: off

doc/help/commands.asciidoc

@@ -544,7 +544,7 @@ For `increment` and `decrement`, the number to change the URL by. For `up`, the
 
 [[open]]
 === open
-Syntax: +:open [*--implicit*] [*--bg*] [*--tab*] [*--window*] ['url']+
+Syntax: +:open [*--implicit*] [*--bg*] [*--tab*] [*--window*] [*--secure*] ['url']+
 
 Open a URL in the current/[count]th tab.
 
@@ -559,6 +559,7 @@ If the URL contains newlines, each line gets opened in its own tab.
 * +*-b*+, +*--bg*+: Open in a new background tab.
 * +*-t*+, +*--tab*+: Open in a new tab.
 * +*-w*+, +*--window*+: Open in a new window.
+* +*-s*+, +*--secure*+: Force HTTPS.
 
 ==== count
 The tab index to open the URL in.
@@ -1598,7 +1599,8 @@ Syntax: +:debug-log-filter 'filters'+
 Change the log filter for console logging.
 
 ==== positional arguments
-* +'filters'+: A comma separated list of logger names.
+* +'filters'+: A comma separated list of logger names. Can also be "none" to clear any existing filters.
+
 
 [[debug-log-level]]
 === debug-log-level
@@ -1653,7 +1655,7 @@ Syntax: +:debug-webaction 'action'+
 
 Execute a webaction.
 
-See http://doc.qt.io/qt-5/qwebpage.html#WebAction-enum for the available actions.
+Available actions: http://doc.qt.io/archives/qt-5.5/qwebpage.html#WebAction-enum (WebKit) http://doc.qt.io/qt-5/qwebenginepage.html#WebAction-enum (WebEngine)
 
 ==== positional arguments
 * +'action'+: The action to execute, e.g. MoveToNextChar.

doc/help/settings.asciidoc

@@ -36,6 +36,7 @@
 [options="header",width="75%",cols="25%,75%"]
 |==============
 |Setting|Description
+|<<ui-history-session-interval,history-session-interval>>|The maximum time in minutes between two history items for them to be considered being from the same session. Use -1 to disable separation.
 |<<ui-zoom-levels,zoom-levels>>|The available zoom levels, separated by commas.
 |<<ui-default-zoom,default-zoom>>|The default zoom level.
 |<<ui-downloads-position,downloads-position>>|Where to show the downloaded files.
@@ -56,6 +57,7 @@
 |<<ui-modal-js-dialog,modal-js-dialog>>|Use standard JavaScript modal dialog for alert() and confirm()
 |<<ui-hide-wayland-decoration,hide-wayland-decoration>>|Hide the window decoration when using wayland (requires restart)
 |<<ui-keyhint-blacklist,keyhint-blacklist>>|Keychains that shouldn't be shown in the keyhint dialog
+|<<ui-keyhint-delay,keyhint-delay>>|Time from pressing a key to seeing the keyhint dialog (ms)
 |<<ui-prompt-radius,prompt-radius>>|The rounding radius for the edges of prompts.
 |<<ui-prompt-filebrowser,prompt-filebrowser>>|Show a filebrowser in upload/download prompts.
 |==============
@@ -124,6 +126,7 @@
 |<<tabs-close-mouse-button,close-mouse-button>>|On which mouse button to close tabs.
 |<<tabs-position,position>>|The position of the tab bar.
 |<<tabs-show-favicons,show-favicons>>|Whether to show favicons in the tab bar.
+|<<tabs-favicon-scale,favicon-scale>>|Scale for favicons in the tab bar. The tab size is unchanged, so big favicons also require extra `tabs->padding`.
 |<<tabs-width,width>>|The width of the tab bar if it's vertical, in px or as percentage of the window.
 |<<tabs-indicator-width,indicator-width>>|Width of the progress indicator (0 to disable).
 |<<tabs-tabs-are-windows,tabs-are-windows>>|Whether to open windows instead of tabs.
@@ -172,7 +175,7 @@
 |<<content-local-content-can-access-remote-urls,local-content-can-access-remote-urls>>|Whether locally loaded documents are allowed to access remote urls.
 |<<content-local-content-can-access-file-urls,local-content-can-access-file-urls>>|Whether locally loaded documents are allowed to access other local urls.
 |<<content-cookies-accept,cookies-accept>>|Control which cookies to accept.
-|<<content-cookies-store,cookies-store>>|Whether to store cookies. Note this option needs a restart with QtWebEngine.
+|<<content-cookies-store,cookies-store>>|Whether to store cookies. Note this option needs a restart with QtWebEngine on Qt < 5.9.
 |<<content-host-block-lists,host-block-lists>>|List of URLs of lists which contain hosts to block.
 |<<content-host-blocking-enabled,host-blocking-enabled>>|Whether host blocking is enabled.
 |<<content-host-blocking-whitelist,host-blocking-whitelist>>|List of domains that should always be loaded, despite being ad-blocked.
@@ -536,6 +539,12 @@ Default: +pass:[path,query]+
 == ui
 General options related to the user interface.
 
+[[ui-history-session-interval]]
+=== history-session-interval
+The maximum time in minutes between two history items for them to be considered being from the same session. Use -1 to disable separation.
+
+Default: +pass:[30]+
+
 [[ui-zoom-levels]]
 === zoom-levels
 The available zoom levels, separated by commas.
@@ -573,6 +582,7 @@ Default: +pass:[bottom]+
 [[ui-message-timeout]]
 === message-timeout
 Time (in ms) to show messages in the statusbar for.
+Set to 0 to never clear messages.
 
 Default: +pass:[2000]+
 
@@ -732,6 +742,12 @@ Globs are supported, so ';*' will blacklist all keychainsstarting with ';'. Use
 
 Default: empty
 
+[[ui-keyhint-delay]]
+=== keyhint-delay
+Time from pressing a key to seeing the keyhint dialog (ms)
+
+Default: +pass:[500]+
+
 [[ui-prompt-radius]]
 === prompt-radius
 The rounding radius for the edges of prompts.
@@ -1191,6 +1207,12 @@ Valid values:
 
 Default: +pass:[true]+
 
+[[tabs-favicon-scale]]
+=== favicon-scale
+Scale for favicons in the tab bar. The tab size is unchanged, so big favicons also require extra `tabs->padding`.
+
+Default: +pass:[1.0]+
+
 [[tabs-width]]
 === width
 The width of the tab bar if it's vertical, in px or as percentage of the window.
@@ -1594,7 +1616,7 @@ This setting is only available with the QtWebKit backend.
 
 [[content-cookies-store]]
 === cookies-store
-Whether to store cookies. Note this option needs a restart with QtWebEngine.
+Whether to store cookies. Note this option needs a restart with QtWebEngine on Qt < 5.9.
 
 Valid values:
 
@@ -2252,7 +2274,7 @@ Fonts used for the UI, with optional style/weight/size.
 === _monospace
 Default monospace fonts.
 
-Default: +pass:[Terminus, Monospace, &quot;DejaVu Sans Mono&quot;, Monaco, &quot;Bitstream Vera Sans Mono&quot;, &quot;Andale Mono&quot;, &quot;Courier New&quot;, Courier, &quot;Liberation Mono&quot;, monospace, Fixed, Consolas, Terminal]+
+Default: +pass:[xos4 Terminus, Terminus, Monospace, &quot;DejaVu Sans Mono&quot;, Monaco, &quot;Bitstream Vera Sans Mono&quot;, &quot;Andale Mono&quot;, &quot;Courier New&quot;, Courier, &quot;Liberation Mono&quot;, monospace, Fixed, Consolas, Terminal]+
 
 [[fonts-completion]]
 === completion

doc/qutebrowser.1.asciidoc

@@ -93,12 +93,6 @@ show it.
 *--nowindow*::
     Don't show the main window.
 
-*--debug-exit*::
-    Turn on debugging of late exit.
-
-*--pdb-postmortem*::
-    Drop into pdb on exceptions.
-
 *--temp-basedir*::
     Use a temporary basedir.
 
@@ -110,6 +104,9 @@ show it.
 
 *--qt-flag* 'QT_FLAG'::
     Pass an argument to Qt as flag.
+
+*--debug-flag* 'DEBUG_FLAGS'::
+    Pass name of debugging feature to be turned on.
 // QUTE_OPTIONS_END
 
 == FILES

misc/requirements/requirements-codecov.txt

@@ -1,5 +1,5 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-codecov==2.0.5
+codecov==2.0.9
-coverage==4.3.4
+coverage==4.4
-requests==2.13.0
+requests==2.14.1

misc/requirements/requirements-flake8.txt

@@ -4,7 +4,7 @@ flake8==2.6.2  # rq.filter: < 3.0.0
 flake8-copyright==0.2.0
 flake8-debugger==1.4.0  # rq.filter: != 2.0.0
 flake8-deprecated==1.1
-flake8-docstrings==1.0.3
+flake8-docstrings==1.0.3  # rq.filter: < 1.1.0
 flake8-future-import==0.4.3
 flake8-mock==0.3
 flake8-pep3101==1.0
@@ -12,9 +12,9 @@ flake8-polyfill==1.0.1
 flake8-putty==0.4.0
 flake8-string-format==0.2.3
 flake8-tidy-imports==1.0.6
-flake8-tuple==0.2.12
+flake8-tuple==0.2.13
 mccabe==0.6.1
 pep8-naming==0.4.1
 pycodestyle==2.3.1
-pydocstyle==1.1.1
+pydocstyle==1.1.1  # rq.filter: < 2.0.0
 pyflakes==1.5.0

misc/requirements/requirements-flake8.txt-raw

@@ -2,7 +2,7 @@ flake8<3.0.0
 flake8-copyright
 flake8-debugger!=2.0.0
 flake8-deprecated
-flake8-docstrings
+flake8-docstrings<1.1.0
 flake8-future-import
 flake8-mock
 flake8-pep3101
@@ -11,7 +11,7 @@ flake8-string-format
 flake8-tidy-imports
 flake8-tuple
 pep8-naming
-pydocstyle
+pydocstyle<2.0.0
 pyflakes
 
 # Pinned to 2.0.0 otherwise
@@ -21,6 +21,8 @@ mccabe==0.6.1
 
 # Waiting until flake8-putty updated
 #@ filter: flake8 < 3.0.0
+#@ filter: pydocstyle < 2.0.0
+#@ filter: flake8-docstrings < 1.1.0
 
 # https://github.com/JBKahn/flake8-debugger/issues/5
 #@ filter: flake8-debugger != 2.0.0

misc/requirements/requirements-pip.txt

@@ -3,6 +3,6 @@
 appdirs==1.4.3
 packaging==16.8
 pyparsing==2.2.0
-setuptools==34.3.2
+setuptools==35.0.2
 six==1.10.0
 wheel==0.29.0

misc/requirements/requirements-pylint-master.txt

@@ -2,10 +2,13 @@
 
 -e git+https://github.com/PyCQA/astroid.git#egg=astroid
 editdistance==0.3.1
+github3.py==0.9.6
 isort==4.2.5
-lazy-object-proxy==1.2.2
+lazy-object-proxy==1.3.1
 mccabe==0.6.1
 -e git+https://github.com/PyCQA/pylint.git#egg=pylint
 ./scripts/dev/pylint_checkers
-requests==2.13.0
+requests==2.14.1
+uritemplate==3.0.0
+uritemplate.py==3.0.2
 wrapt==1.10.10

misc/requirements/requirements-pylint-master.txt-raw

@@ -2,6 +2,7 @@
 -e git+https://github.com/PyCQA/pylint.git#egg=pylint
 ./scripts/dev/pylint_checkers
 requests
+github3.py
 
 # remove @commit-id for scm installs
 #@ replace: @.*# #

misc/requirements/requirements-pylint.txt

@@ -1,13 +1,13 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-astroid==1.4.9
+astroid==1.5.2
 github3.py==0.9.6
 isort==4.2.5
-lazy-object-proxy==1.2.2
+lazy-object-proxy==1.3.1
 mccabe==0.6.1
-pylint==1.6.5
+pylint==1.7.1
 ./scripts/dev/pylint_checkers
-requests==2.13.0
+requests==2.14.1
 uritemplate==3.0.0
 uritemplate.py==3.0.2
 wrapt==1.10.10

misc/requirements/requirements-pyqt.txt

@@ -1,4 +1,4 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-PyQt5==5.8.1.1
+PyQt5==5.8.2
-sip==4.19.1
+sip==4.19.2

misc/requirements/requirements-tests.txt

@@ -1,15 +1,15 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-beautifulsoup4==4.5.3
+beautifulsoup4==4.6.0
-cheroot==5.3.0
+cheroot==5.5.0
 click==6.7
-coverage==4.3.4
+coverage==4.4
 decorator==4.0.11
 EasyProcess==0.2.3
-Flask==0.12
+Flask==0.12.1
 glob2==0.5
 httpbin==0.5.0
-hypothesis==3.6.1
+hypothesis==3.8.3
 itsdangerous==0.24
 # Jinja2==2.9.5
 Mako==1.0.6
@@ -18,13 +18,13 @@ parse==1.8.0
 parse-type==0.3.4
 py==1.4.33
 pytest==3.0.7
-pytest-bdd==2.18.1
+pytest-bdd==2.18.2
 pytest-benchmark==3.0.0
 pytest-catchlog==1.2.2
-pytest-cov==2.4.0
+pytest-cov==2.5.0
 pytest-faulthandler==1.3.1
 pytest-instafail==0.3.0
-pytest-mock==1.5.0
+pytest-mock==1.6.0
 pytest-qt==2.1.0
 pytest-repeat==0.4.1
 pytest-rerunfailures==2.1.0
@@ -32,5 +32,5 @@ pytest-travis-fold==1.2.0
 pytest-warnings==0.2.0
 pytest-xvfb==1.0.0
 PyVirtualDisplay==0.2.1
-vulture==0.13
+vulture==0.14
 Werkzeug==0.12.1

misc/requirements/requirements-tox.txt

@@ -2,5 +2,5 @@
 
 pluggy==0.4.0
 py==1.4.33
-tox==2.6.0
+tox==2.7.0
 virtualenv==15.1.0

misc/requirements/requirements-vulture.txt

@@ -1,3 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-vulture==0.13
+vulture==0.14

misc/userscripts/dmenu_qutebrowser

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 # Copyright 2015 Zach-Button <zachrey.button@gmail.com>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

misc/userscripts/openfeeds

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # Copyright 2015 jnphilipp <me@jnphilipp.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

misc/userscripts/password_fill

@@ -9,7 +9,7 @@ directly ask me via IRC (nickname thorsten\`) in #qutebrowser on freenode.
 
   $blink!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!$reset
   WARNING: the passwords are stored in qutebrowser's
-           debug log reachable via the url qute:log
+           debug log reachable via the url qute://log
   $blink!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!$reset
 
 Usage: run as a userscript form qutebrowser, e.g.:

misc/userscripts/qutebrowser_viewsource

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 # Copyright 2015 Zach-Button <zachrey.button@gmail.com>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

pytest.ini

@@ -24,11 +24,12 @@ markers =
     js_prompt: Tests needing to display a javascript prompt
     this: Used to mark tests during development
     no_invalid_lines: Don't fail on unparseable lines in end2end tests
+    issue2478: Tests which are broken on Windows with QtWebEngine, https://github.com/qutebrowser/qutebrowser/issues/2478
 qt_log_level_fail = WARNING
 qt_log_ignore =
     ^SpellCheck: .*
     ^SetProcessDpiAwareness failed: .*
-    ^QWindowsWindow::setGeometryDp: Unable to set geometry .*
+    ^QWindowsWindow::setGeometry(Dp)?: Unable to set geometry .*
     ^QProcess: Destroyed while process .* is still running\.
     ^"Method "GetAll" with signature "s" on interface "org\.freedesktop\.DBus\.Properties" doesn't exist
     ^"Method \\"GetAll\\" with signature \\"s\\" on interface \\"org\.freedesktop\.DBus\.Properties\\" doesn't exist\\n"
@@ -51,4 +52,5 @@ qt_log_ignore =
     ^Image of format '' blocked because it is not considered safe. If you are sure it is safe to do so, you can white-list the format by setting the environment variable QTWEBKIT_IMAGEFORMAT_WHITELIST=
     ^QPainter::end: Painter ended with \d+ saved states
     ^QSslSocket: cannot resolve SSLv[23]_(client|server)_method
+    ^QQuickWidget::invalidateRenderControl could not make context current
 xfail_strict = true

qutebrowser/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,7 +22,7 @@
 import os.path
 
 __author__ = "Florian Bruhin"
-__copyright__ = "Copyright 2014-2016 Florian Bruhin (The Compiler)"
+__copyright__ = "Copyright 2014-2017 Florian Bruhin (The Compiler)"
 __license__ = "GPL"
 __maintainer__ = __author__
 __email__ = "mail@qutebrowser.org"

qutebrowser/__main__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/app.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -170,11 +170,14 @@ def _init_icon():
     for size in [16, 24, 32, 48, 64, 96, 128, 256, 512]:
         filename = ':/icons/qutebrowser-{}x{}.png'.format(size, size)
         pixmap = QPixmap(filename)
-        qtutils.ensure_not_null(pixmap)
+        if pixmap.isNull():
+            log.init.warning("Failed to load {}".format(filename))
+        else:
             fallback_icon.addPixmap(pixmap)
-    qtutils.ensure_not_null(fallback_icon)
     icon = QIcon.fromTheme('qutebrowser', fallback_icon)
-    qtutils.ensure_not_null(icon)
+    if icon.isNull():
+        log.init.warning("Failed to load icon")
+    else:
         qApp.setWindowIcon(icon)
 
 
@@ -301,7 +304,7 @@ def _open_startpage(win_id=None):
         window_ids = [win_id]
     else:
         window_ids = objreg.window_registry
-    for cur_win_id in window_ids:
+    for cur_win_id in list(window_ids):  # Copying as the dict could change
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window=cur_win_id)
         if tabbed_browser.count() == 0:
@@ -340,7 +343,8 @@ def _open_quickstart(args):
 
 def _save_version():
     """Save the current version to the state config."""
-    state_config = objreg.get('state-config')
+    state_config = objreg.get('state-config', None)
+    if state_config is not None:
         state_config['general']['version'] = qutebrowser.__version__
 
 
@@ -647,8 +651,8 @@ class Quitter:
         self._shutting_down = True
         log.destroy.debug("Shutting down with status {}, session {}...".format(
             status, session))
-
+        session_manager = objreg.get('session-manager', None)
-        session_manager = objreg.get('session-manager')
+        if session_manager is not None:
             if session is not None:
                 session_manager.save(session, last_window=last_window,
                                      load_next_time=True)
@@ -671,7 +675,7 @@ class Quitter:
             # event loop, so we can shut down immediately.
             self._shutdown(status, restart=restart)
 
-    def _shutdown(self, status, restart):
+    def _shutdown(self, status, restart):  # noqa
         """Second stage of shutdown."""
         log.destroy.debug("Stage 2 of shutting down...")
         if qApp is None:
@@ -680,7 +684,9 @@ class Quitter:
         # Remove eventfilter
         try:
             log.destroy.debug("Removing eventfilter...")
-            qApp.removeEventFilter(objreg.get('event-filter'))
+            event_filter = objreg.get('event-filter', None)
+            if event_filter is not None:
+                qApp.removeEventFilter(event_filter)
         except AttributeError:
             pass
         # Close all windows
@@ -722,7 +728,9 @@ class Quitter:
         # Now we can hopefully quit without segfaults
         log.destroy.debug("Deferring QApplication::exit...")
         objreg.get('signal-handler').deactivate()
-        objreg.get('session-manager').delete_autosave()
+        session_manager = objreg.get('session-manager', None)
+        if session_manager is not None:
+            session_manager.delete_autosave()
         # We use a singleshot timer to exit here to minimize the likelihood of
         # segfaults.
         QTimer.singleShot(0, functools.partial(qApp.exit, status))
@@ -784,7 +792,7 @@ class Application(QApplication):
     def exit(self, status):
         """Extend QApplication::exit to log the event."""
         log.destroy.debug("Now calling QApplication::exit.")
-        if self._args.debug_exit:
+        if 'debug-exit' in self._args.debug_flags:
             if hunter is None:
                 print("Not logging late shutdown because hunter could not be "
                       "imported!", file=sys.stderr)

qutebrowser/browser/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/adblock.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/browsertab.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -21,7 +21,7 @@
 
 import itertools
 
-from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QSizeF
+from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QSizeF, Qt
 from PyQt5.QtGui import QIcon
 from PyQt5.QtWidgets import QWidget, QApplication
 
@@ -107,7 +107,15 @@ class TabData:
 
 class AbstractAction:
 
-    """Attribute of AbstractTab for Qt WebActions."""
+    """Attribute of AbstractTab for Qt WebActions.
+
+    Class attributes (overridden by subclasses):
+        action_class: The class actions are defined on (QWeb{Engine,}Page)
+        action_base: The type of the actions (QWeb{Engine,}Page.WebAction)
+    """
+
+    action_class = None
+    action_base = None
 
     def __init__(self):
         self._widget = None
@@ -120,6 +128,13 @@ class AbstractAction:
         """Save the current page."""
         raise NotImplementedError
 
+    def run_string(self, name):
+        """Run a webaction based on its name."""
+        member = getattr(self.action_class, name, None)
+        if not isinstance(member, self.action_base):
+            raise WebTabError("{} is not a valid web action!".format(name))
+        self._widget.triggerPageAction(member)
+
 
 class AbstractPrinting:
 
@@ -157,6 +172,8 @@ class AbstractSearch(QObject):
 
     Attributes:
         text: The last thing this view was searched for.
+        search_displayed: Whether we're currently displaying search results in
+                          this view.
         _flags: The flags of the last search (needs to be set by subclasses).
         _widget: The underlying WebView widget.
     """
@@ -165,6 +182,7 @@ class AbstractSearch(QObject):
         super().__init__(parent)
         self._widget = None
         self.text = None
+        self.search_displayed = False
 
     def search(self, text, *, ignore_case=False, reverse=False,
                result_cb=None):
@@ -742,6 +760,10 @@ class AbstractTab(QWidget):
     def clear_ssl_errors(self):
         raise NotImplementedError
 
+    def key_press(self, key, modifier=Qt.NoModifier):
+        """Send a fake key event to this tab."""
+        raise NotImplementedError
+
     def dump_async(self, callback, *, plain=False):
         """Dump the current page to a file ascync.
 

qutebrowser/browser/commands.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -28,14 +28,6 @@ from PyQt5.QtWidgets import QApplication, QTabBar
 from PyQt5.QtCore import Qt, QUrl, QEvent, QUrlQuery
 from PyQt5.QtGui import QKeyEvent
 from PyQt5.QtPrintSupport import QPrintDialog, QPrintPreviewDialog
-try:
-    from PyQt5.QtWebKitWidgets import QWebPage
-except ImportError:
-    QWebPage = None
-try:
-    from PyQt5.QtWebEngineWidgets import QWebEnginePage
-except ImportError:
-    QWebEnginePage = None
 import pygments
 import pygments.lexers
 import pygments.formatters
@@ -289,7 +281,7 @@ class CommandDispatcher:
     @cmdutils.argument('url', completion=usertypes.Completion.url)
     @cmdutils.argument('count', count=True)
     def openurl(self, url=None, implicit=False,
-                bg=False, tab=False, window=False, count=None):
+                bg=False, tab=False, window=False, count=None, secure=False):
         """Open a URL in the current/[count]th tab.
 
         If the URL contains newlines, each line gets opened in its own tab.
@@ -302,6 +294,7 @@ class CommandDispatcher:
             implicit: If opening a new tab, treat the tab as implicit (like
                       clicking on a link).
             count: The tab index to open the URL in, or None.
+            secure: Force HTTPS.
         """
         if url is None:
             urls = [config.get('general', 'default-page')]
@@ -309,6 +302,8 @@ class CommandDispatcher:
             urls = self._parse_url_input(url)
 
         for i, cur_url in enumerate(urls):
+            if secure:
+                cur_url.setScheme('https')
             if not window and i > 0:
                 tab = False
                 bg = True
@@ -1285,7 +1280,7 @@ class CommandDispatcher:
         except urlmarks.Error as e:
             raise cmdexc.CommandError(str(e))
         else:
-            msg = "Bookmarked {}!" if was_added else "Removed bookmark {}!"
+            msg = "Bookmarked {}" if was_added else "Removed bookmark {}"
             message.info(msg.format(url.toDisplayString()))
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
@@ -1389,6 +1384,9 @@ class CommandDispatcher:
                                       scope='window', window=self._win_id)
         target = None
         if dest is not None:
+            dest = downloads.transform_path(dest)
+            if dest is None:
+                raise cmdexc.CommandError("Invalid target filename")
             target = downloads.FileDownloadTarget(dest)
 
         tab = self._current_widget()
@@ -1554,6 +1552,7 @@ class CommandDispatcher:
         if text is None:
             message.error("Could not get text from the focused element.")
             return
+        assert isinstance(text, str), text
 
         ed = editor.ExternalEditor(self._tabbed_browser)
         ed.editing_finished.connect(functools.partial(
@@ -1591,10 +1590,7 @@ class CommandDispatcher:
                        backend=usertypes.Backend.QtWebKit)
     def paste_primary(self):
         """Paste the primary selection at cursor position."""
-        try:
+        self.insert_text(utils.get_clipboard(selection=True, fallback=True))
-            self.insert_text(utils.get_clipboard(selection=True))
-        except utils.SelectionUnsupportedError:
-            self.insert_text(utils.get_clipboard())
 
     @cmdutils.register(instance='command-dispatcher', maxsplit=0,
                        scope='window')
@@ -1705,21 +1701,22 @@ class CommandDispatcher:
         tab = self._current_widget()
         tab.search.clear()
 
+        if not text:
+            return
+
         options = {
             'ignore_case': config.get('general', 'ignore-case'),
             'reverse': reverse,
         }
+
         self._tabbed_browser.search_text = text
         self._tabbed_browser.search_options = dict(options)
 
-        if text:
         cb = functools.partial(self._search_cb, tab=tab,
                                old_scroll_pos=tab.scroller.pos_px(),
                                options=options, text=text, prev=False)
-        else:
-            cb = None
-
         options['result_cb'] = cb
+
         tab.search.search(text, **options)
 
     @cmdutils.register(instance='command-dispatcher', hide=True,
@@ -1955,33 +1952,20 @@ class CommandDispatcher:
     def debug_webaction(self, action, count=1):
         """Execute a webaction.
 
-        See http://doc.qt.io/qt-5/qwebpage.html#WebAction-enum for the
+        Available actions:
-        available actions.
+        http://doc.qt.io/archives/qt-5.5/qwebpage.html#WebAction-enum (WebKit)
+        http://doc.qt.io/qt-5/qwebenginepage.html#WebAction-enum (WebEngine)
 
         Args:
             action: The action to execute, e.g. MoveToNextChar.
             count: How many times to repeat the action.
         """
         tab = self._current_widget()
-
-        if tab.backend == usertypes.Backend.QtWebKit:
-            assert QWebPage is not None
-            member = getattr(QWebPage, action, None)
-            base = QWebPage.WebAction
-        elif tab.backend == usertypes.Backend.QtWebEngine:
-            assert QWebEnginePage is not None
-            member = getattr(QWebEnginePage, action, None)
-            base = QWebEnginePage.WebAction
-
-        if not isinstance(member, base):
-            raise cmdexc.CommandError("{} is not a valid web action!".format(
-                action))
-
         for _ in range(count):
-            # This whole command is backend-specific anyways, so it makes no
+            try:
-            # sense to introduce some API for this.
+                tab.action.run_string(action)
-            # pylint: disable=protected-access
+            except browsertab.WebTabError as e:
-            tab._widget.triggerPageAction(member)
+                raise cmdexc.CommandError(str(e))
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0, no_cmd_split=True)

qutebrowser/browser/downloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,11 +19,13 @@
 
 """Shared QtWebKit/QtWebEngine code for downloads."""
 
+import re
 import sys
 import html
 import os.path
 import collections
 import functools
+import pathlib
 import tempfile
 
 import sip
@@ -161,6 +163,25 @@ def get_filename_question(*, suggested_filename, url, parent=None):
     return q
 
 
+def transform_path(path):
+    r"""Do platform-specific transformations, like changing E: to E:\.
+
+    Returns None if the path is invalid on the current platform.
+    """
+    if sys.platform != "win32":
+        return path
+    path = utils.expand_windows_drive(path)
+    # Drive dependent working directories are not supported, e.g.
+    # E:filename is invalid
+    if re.match(r'[A-Z]:[^\\]', path, re.IGNORECASE):
+        return None
+    # Paths like COM1, ...
+    # See https://github.com/qutebrowser/qutebrowser/issues/82
+    if pathlib.Path(path).is_reserved():
+        return None
+    return path
+
+
 class NoFilenameError(Exception):
 
     """Raised when we can't find out a filename in DownloadTarget."""
@@ -507,6 +528,14 @@ class AbstractDownloadItem(QObject):
         """Retry a failed download."""
         raise NotImplementedError
 
+    @pyqtSlot()
+    def try_retry(self):
+        """Try to retry a download and show an error if it's unsupported."""
+        try:
+            self.retry()
+        except UnsupportedOperationError as e:
+            message.error(str(e))
+
     def _get_open_filename(self):
         """Get the filename to open a download.
 
@@ -968,7 +997,7 @@ class DownloadModel(QAbstractListModel):
                 raise cmdexc.CommandError("No failed downloads!")
             else:
                 download = to_retry[0]
-        download.retry()
+        download.try_retry()
 
     def can_clear(self):
         """Check if there are finished downloads to clear."""

qutebrowser/browser/downloadview.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -23,7 +23,7 @@ import functools
 
 import sip
 from PyQt5.QtCore import pyqtSlot, QSize, Qt, QTimer
-from PyQt5.QtWidgets import QListView, QSizePolicy, QMenu
+from PyQt5.QtWidgets import QListView, QSizePolicy, QMenu, QStyleFactory
 
 from qutebrowser.browser import downloads
 from qutebrowser.config import style
@@ -75,6 +75,7 @@ class DownloadView(QListView):
 
     def __init__(self, win_id, parent=None):
         super().__init__(parent)
+        self.setStyle(QStyleFactory.create('Fusion'))
         style.set_register_stylesheet(self)
         self.setResizeMode(QListView.Adjust)
         self.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
@@ -134,7 +135,7 @@ class DownloadView(QListView):
             if item.successful:
                 actions.append(("Open", item.open_file))
             else:
-                actions.append(("Retry", item.retry))
+                actions.append(("Retry", item.try_retry))
             actions.append(("Remove", item.remove))
         else:
             actions.append(("Cancel", item.cancel))

qutebrowser/browser/hints.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/history.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/inspector.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/mouse.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/navigate.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/network/pac.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -155,7 +155,7 @@ class PACResolver:
                 raise ParseProxyError("Invalid number of parameters for PROXY")
             host, port = PACResolver._parse_proxy_host(config[1])
             return QNetworkProxy(QNetworkProxy.HttpProxy, host, port)
-        elif config[0] == "SOCKS":
+        elif config[0] in ["SOCKS", "SOCKS5"]:
             if len(config) != 2:
                 raise ParseProxyError("Invalid number of parameters for SOCKS")
             host, port = PACResolver._parse_proxy_host(config[1])

qutebrowser/browser/network/proxy.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/pdfjs.py

@@ -1,6 +1,7 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
 # Copyright 2015 Daniel Schadt
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/qtnetworkdownloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -110,6 +110,9 @@ class DownloadItem(downloads.AbstractDownloadItem):
     def _do_die(self):
         """Abort the download and emit an error."""
         self._read_timer.stop()
+        if self._reply is None:
+            log.downloads.debug("Reply gone while dying")
+            return
         self._reply.downloadProgress.disconnect()
         self._reply.finished.disconnect()
         self._reply.error.disconnect()
@@ -270,7 +273,7 @@ class DownloadItem(downloads.AbstractDownloadItem):
         if self.fileobj is None or self._reply is None:
             # No filename has been set yet (so we don't empty the buffer) or we
             # got a readyRead after the reply was finished (which happens on
-            # qute:log for example).
+            # qute://log for example).
             return
         if not self._reply.isOpen():
             raise OSError("Reply is closed!")

qutebrowser/browser/qutescheme.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,23 +17,26 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Backend-independent qute:* code.
+"""Backend-independent qute://* code.
 
 Module attributes:
     pyeval_output: The output of the last :pyeval command.
     _HANDLERS: The handlers registered via decorators.
 """
 
+import json
+import os
 import sys
 import time
-import datetime
 import urllib.parse
+import datetime
 
-from PyQt5.QtCore import QUrlQuery
+from PyQt5.QtCore import QUrlQuery, QUrl
 
 import qutebrowser
+from qutebrowser.config import config
 from qutebrowser.utils import (version, utils, jinja, log, message, docutils,
-                               objreg)
+                               objreg, usertypes, qtutils)
 from qutebrowser.misc import objects
 
 
@@ -75,12 +78,25 @@ class QuteSchemeError(Exception):
         super().__init__(errorstring)
 
 
-class add_handler:  # pylint: disable=invalid-name
+class Redirect(Exception):
 
-    """Decorator to register a qute:* URL handler.
+    """Exception to signal a redirect should happen.
 
     Attributes:
-        _name: The 'foo' part of qute:foo
+        url: The URL to redirect to, as a QUrl.
+    """
+
+    def __init__(self, url):
+        super().__init__(url.toDisplayString())
+        self.url = url
+
+
+class add_handler:  # pylint: disable=invalid-name
+
+    """Decorator to register a qute://* URL handler.
+
+    Attributes:
+        _name: The 'foo' part of qute://foo
         backend: Limit which backends the handler can run with.
     """
 
@@ -103,7 +119,7 @@ class add_handler:  # pylint: disable=invalid-name
     def wrong_backend_handler(self, url):
         """Show an error page about using the invalid backend."""
         html = jinja.render('error.html',
-                            title="Error while opening qute:url",
+                            title="Error while opening qute://url",
                             url=url.toDisplayString(),
                             error='{} is not available with this '
                                   'backend'.format(url.toDisplayString()),
@@ -125,13 +141,19 @@ def data_for_url(url):
     # A url like "qute:foo" is split as "scheme:path", not "scheme:host".
     log.misc.debug("url: {}, path: {}, host {}".format(
         url.toDisplayString(), path, host))
-    try:
+    if path and not host:
-        handler = _HANDLERS[path]
+        new_url = QUrl()
-    except KeyError:
+        new_url.setScheme('qute')
+        new_url.setHost(path)
+        new_url.setPath('/')
+        if new_url.host():  # path was a valid host
+            raise Redirect(new_url)
+
     try:
         handler = _HANDLERS[host]
     except KeyError:
         raise NoHandlerFound(url)
+
     try:
         mimetype, data = handler(url)
     except OSError as e:
@@ -150,7 +172,7 @@ def data_for_url(url):
 
 @add_handler('bookmarks')
 def qute_bookmarks(_url):
-    """Handler for qute:bookmarks. Display all quickmarks / bookmarks."""
+    """Handler for qute://bookmarks. Display all quickmarks / bookmarks."""
     bookmarks = sorted(objreg.get('bookmark-manager').marks.items(),
                        key=lambda x: x[1])  # Sort by title
     quickmarks = sorted(objreg.get('quickmark-manager').marks.items(),
@@ -163,90 +185,160 @@ def qute_bookmarks(_url):
     return 'text/html', html
 
 
-@add_handler('history')  # noqa
+def history_data(start_time):  # noqa
-def qute_history(url):
+    """Return history data
-    """Handler for qute:history. Display history."""
-    # Get current date from query parameter, if not given choose today.
-    curr_date = datetime.date.today()
-    try:
-        query_date = QUrlQuery(url).queryItemValue("date")
-        if query_date:
-            curr_date = datetime.datetime.strptime(query_date, "%Y-%m-%d")
-            curr_date = curr_date.date()
-    except ValueError:
-        log.misc.debug("Invalid date passed to qute:history: " + query_date)
 
-    one_day = datetime.timedelta(days=1)
+    Arguments:
-    next_date = curr_date + one_day
+        start_time -- select history starting from this timestamp.
-    prev_date = curr_date - one_day
+    """
+    def history_iter(start_time, reverse=False):
+        """Iterate through the history and get items we're interested.
 
-    def history_iter(reverse):
+        Arguments:
-        """Iterate through the history and get items we're interested in."""
+            reverse -- whether to reverse the history_dict before iterating.
-        curr_timestamp = time.mktime(curr_date.timetuple())
+        """
         history = objreg.get('web-history').history_dict.values()
         if reverse:
             history = reversed(history)
 
+        # when history_dict is not reversed, we need to keep track of last item
+        # so that we can yield its atime
+        last_item = None
+
+        # end is 24hrs earlier than start
+        end_time = start_time - 24*60*60
+
         for item in history:
-            # If we can't apply the reverse performance trick below,
-            # at least continue as early as possible with old items.
-            # This gets us down from 550ms to 123ms with 500k old items on my
-            # machine.
-            if item.atime < curr_timestamp and not reverse:
-                continue
-
-            # Convert timestamp
-            try:
-                item_atime = datetime.datetime.fromtimestamp(item.atime)
-            except (ValueError, OSError, OverflowError):
-                log.misc.debug("Invalid timestamp {}.".format(item.atime))
-                continue
-
-            if reverse and item_atime.date() < curr_date:
-                # If we could reverse the history in-place, and this entry is
-                # older than today, only older entries will follow, so we can
-                # abort here.
-                return
-
-            # Skip items not on curr_date
             # Skip redirects
             # Skip qute:// links
-            is_internal = item.url.scheme() == 'qute'
+            if item.redirect or item.url.scheme() == 'qute':
-            is_not_today = item_atime.date() != curr_date
-            if item.redirect or is_internal or is_not_today:
                 continue
 
+            # Skip items out of time window
+            item_newer = item.atime > start_time
+            item_older = item.atime <= end_time
+            if reverse:
+                # history_dict is reversed, we are going back in history.
+                # so:
+                #     abort if item is older than start_time+24hr
+                #     skip if item is newer than start
+                if item_older:
+                    yield {"next": int(item.atime)}
+                    return
+                if item_newer:
+                    continue
+            else:
+                # history_dict isn't reversed, we are going forward in history.
+                # so:
+                #     abort if item is newer than start_time
+                #     skip if item is older than start_time+24hrs
+                if item_older:
+                    last_item = item
+                    continue
+                if item_newer:
+                    yield {"next": int(last_item.atime if last_item else -1)}
+                    return
+
             # Use item's url as title if there's no title.
             item_url = item.url.toDisplayString()
             item_title = item.title if item.title else item_url
-            display_atime = item_atime.strftime("%X")
+            item_time = int(item.atime * 1000)
 
-            yield (item_url, item_title, display_atime)
+            yield {"url": item_url, "title": item_title, "time": item_time}
+
+        # if we reached here, we had reached the end of history
+        yield {"next": int(last_item.atime if last_item else -1)}
 
     if sys.hexversion >= 0x03050000:
         # On Python >= 3.5 we can reverse the ordereddict in-place and thus
         # apply an additional performance improvement in history_iter.
         # On my machine, this gets us down from 550ms to 72us with 500k old
         # items.
-        history = list(history_iter(reverse=True))
+        history = history_iter(start_time, reverse=True)
     else:
         # On Python 3.4, we can't do that, so we'd need to copy the entire
         # history to a list. There, filter first and then reverse it here.
-        history = reversed(list(history_iter(reverse=False)))
+        history = reversed(list(history_iter(start_time, reverse=False)))
 
-    html = jinja.render('history.html',
+    return list(history)
+
+
+@add_handler('history')
+def qute_history(url):
+    """Handler for qute://history. Display and serve history."""
+    if url.path() == '/data':
+        # Use start_time in query or current time.
+        try:
+            start_time = QUrlQuery(url).queryItemValue("start_time")
+            start_time = float(start_time) if start_time else time.time()
+        except ValueError as e:
+            raise QuteSchemeError("Query parameter start_time is invalid", e)
+
+        return 'text/html', json.dumps(history_data(start_time))
+    else:
+        if (
+            config.get('content', 'allow-javascript') and
+            (objects.backend == usertypes.Backend.QtWebEngine or
+             qtutils.is_qtwebkit_ng())
+        ):
+            return 'text/html', jinja.render(
+                'history.html',
+                title='History',
+                session_interval=config.get('ui', 'history-session-interval')
+            )
+        else:
+            # Get current date from query parameter, if not given choose today.
+            curr_date = datetime.date.today()
+            try:
+                query_date = QUrlQuery(url).queryItemValue("date")
+                if query_date:
+                    curr_date = datetime.datetime.strptime(query_date,
+                        "%Y-%m-%d").date()
+            except ValueError:
+                log.misc.debug("Invalid date passed to qute:history: " +
+                    query_date)
+
+            one_day = datetime.timedelta(days=1)
+            next_date = curr_date + one_day
+            prev_date = curr_date - one_day
+
+            # start_time is the last second of curr_date
+            start_time = time.mktime(next_date.timetuple()) - 1
+            history = [
+                (i["url"], i["title"],
+                 datetime.datetime.fromtimestamp(i["time"]/1000),
+                 QUrl(i["url"]).host())
+                for i in history_data(start_time) if "next" not in i
+            ]
+
+            return 'text/html', jinja.render(
+                'history_nojs.html',
                 title='History',
                 history=history,
                 curr_date=curr_date,
                 next_date=next_date,
                 prev_date=prev_date,
-                        today=datetime.date.today())
+                today=datetime.date.today(),
-    return 'text/html', html
+            )
+
+
+@add_handler('javascript')
+def qute_javascript(url):
+    """Handler for qute://javascript.
+
+    Return content of file given as query parameter.
+    """
+    path = url.path()
+    if path:
+        path = "javascript" + os.sep.join(path.split('/'))
+        return 'text/html', utils.read_file(path, binary=False)
+    else:
+        raise QuteSchemeError("No file specified", ValueError())
 
 
 @add_handler('pyeval')
 def qute_pyeval(_url):
-    """Handler for qute:pyeval."""
+    """Handler for qute://pyeval."""
     html = jinja.render('pre.html', title='pyeval', content=pyeval_output)
     return 'text/html', html
 
@@ -254,7 +346,7 @@ def qute_pyeval(_url):
 @add_handler('version')
 @add_handler('verizon')
 def qute_version(_url):
-    """Handler for qute:version."""
+    """Handler for qute://version."""
     html = jinja.render('version.html', title='Version info',
                         version=version.version(),
                         copyright=qutebrowser.__copyright__)
@@ -263,7 +355,7 @@ def qute_version(_url):
 
 @add_handler('plainlog')
 def qute_plainlog(url):
-    """Handler for qute:plainlog.
+    """Handler for qute://plainlog.
 
     An optional query parameter specifies the minimum log level to print.
     For example, qute://log?level=warning prints warnings and errors.
@@ -283,7 +375,7 @@ def qute_plainlog(url):
 
 @add_handler('log')
 def qute_log(url):
-    """Handler for qute:log.
+    """Handler for qute://log.
 
     An optional query parameter specifies the minimum log level to print.
     For example, qute://log?level=warning prints warnings and errors.
@@ -304,13 +396,13 @@ def qute_log(url):
 
 @add_handler('gpl')
 def qute_gpl(_url):
-    """Handler for qute:gpl. Return HTML content as string."""
+    """Handler for qute://gpl. Return HTML content as string."""
     return 'text/html', utils.read_file('html/COPYING.html')
 
 
 @add_handler('help')
 def qute_help(url):
-    """Handler for qute:help."""
+    """Handler for qute://help."""
     try:
         utils.read_file('html/doc/index.html')
     except OSError:

qutebrowser/browser/shared.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/signalfilter.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/urlmarks.py

@@ -1,7 +1,7 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-# Copyright 2015-2016 Antoni Boucher <bouanto@zoho.com>
+# Copyright 2015-2017 Antoni Boucher <bouanto@zoho.com>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webelem.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -306,6 +306,11 @@ class AbstractWebElement(collections.abc.MutableMapping):
         qtutils.ensure_valid(url)
         return url
 
+    def is_link(self):
+        """Return True if this AbstractWebElement is a link."""
+        href_tags = ['a', 'area', 'link']
+        return self.tag_name() in href_tags
+
     def _mouse_pos(self):
         """Get the position to click/hover."""
         # Click the center of the largest square fitting into the top/left
@@ -403,9 +408,8 @@ class AbstractWebElement(collections.abc.MutableMapping):
             self._click_fake_event(click_target)
             return
 
-        href_tags = ['a', 'area', 'link']
         if click_target == usertypes.ClickTarget.normal:
-            if self.tag_name() in href_tags:
+            if self.is_link():
                 log.webelem.debug("Clicking via JS click()")
                 self._click_js(click_target)
             elif self.is_editable(strict=True):
@@ -418,7 +422,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
         elif click_target in [usertypes.ClickTarget.tab,
                               usertypes.ClickTarget.tab_bg,
                               usertypes.ClickTarget.window]:
-            if self.tag_name() in href_tags:
+            if self.is_link():
                 self._click_href(click_target)
             else:
                 self._click_fake_event(click_target)

qutebrowser/browser/webengine/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webengine/certificateerror.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webengine/interceptor.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -57,8 +57,7 @@ class RequestInterceptor(QWebEngineUrlRequestInterceptor):
             info: QWebEngineUrlRequestInfo &info
         """
         # FIXME:qtwebengine only block ads for NavigationTypeOther?
-        if (bytes(info.requestMethod()) == b'GET' and
+        if self._host_blocker.is_blocked(info.requestUrl()):
-                self._host_blocker.is_blocked(info.requestUrl())):
             log.webview.info("Request to {} blocked by host blocker.".format(
                 info.requestUrl().host()))
             info.block(True)

qutebrowser/browser/webengine/tabhistory.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webengine/webenginedownloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -78,16 +78,15 @@ class DownloadItem(downloads.AbstractDownloadItem):
             self.stats.finish()
         elif state == QWebEngineDownloadItem.DownloadInterrupted:
             self.successful = False
-            self.done = True
             # https://bugreports.qt.io/browse/QTBUG-56839
-            self.error.emit("Download failed")
+            self._die("Download failed")
-            self.stats.finish()
         else:
             raise ValueError("_on_state_changed was called with unknown state "
                              "{}".format(state_name))
 
     def _do_die(self):
         self._qt_item.downloadProgress.disconnect()
+        if self._qt_item.state() != QWebEngineDownloadItem.DownloadInterrupted:
             self._qt_item.cancel()
 
     def _do_cancel(self):
@@ -95,15 +94,18 @@ class DownloadItem(downloads.AbstractDownloadItem):
 
     def retry(self):
         # https://bugreports.qt.io/browse/QTBUG-56840
-        raise downloads.UnsupportedOperationError
+        raise downloads.UnsupportedOperationError(
+            "Retrying downloads is unsupported with QtWebEngine")
 
     def _get_open_filename(self):
         return self._filename
 
-    def _set_fileobj(self, fileobj):
+    def _set_fileobj(self, fileobj, *,
+                     autoclose=True):  # pylint: disable=unused-argument
         raise downloads.UnsupportedOperationError
 
     def _set_tempfile(self, fileobj):
+        fileobj.close()
         self._set_filename(fileobj.name, force_overwrite=True,
                            remember_directory=False)
 
@@ -145,7 +147,7 @@ def _get_suggested_filename(path):
     """
     filename = os.path.basename(path)
     filename = re.sub(r'\([0-9]+\)(?=\.|$)', '', filename)
-    if not qtutils.version_check('5.8.1'):
+    if not qtutils.version_check('5.9'):
         # https://bugreports.qt.io/browse/QTBUG-58155
         filename = urllib.parse.unquote(filename)
         # Doing basename a *second* time because there could be a %2F in

qutebrowser/browser/webengine/webengineelem.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -18,7 +18,7 @@
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
 # FIXME:qtwebengine remove this once the stubs are gone
-# pylint: disable=unused-variable
+# pylint: disable=unused-argument
 
 """QtWebEngine specific part of the web element API."""
 
@@ -39,6 +39,38 @@ class WebEngineElement(webelem.AbstractWebElement):
 
     def __init__(self, js_dict, tab):
         super().__init__(tab)
+        # Do some sanity checks on the data we get from JS
+        js_dict_types = {
+            'id': int,
+            'text': str,
+            'value': (str, int, float),
+            'tag_name': str,
+            'outer_xml': str,
+            'class_name': str,
+            'rects': list,
+            'attributes': dict,
+        }
+        assert set(js_dict.keys()).issubset(js_dict_types.keys())
+        for name, typ in js_dict_types.items():
+            if name in js_dict and not isinstance(js_dict[name], typ):
+                raise TypeError("Got {} for {} from JS but expected {}: "
+                                "{}".format(type(js_dict[name]), name, typ,
+                                            js_dict))
+        for name, value in js_dict['attributes'].items():
+            if not isinstance(name, str):
+                raise TypeError("Got {} ({}) for attribute name from JS: "
+                                "{}".format(name, type(name), js_dict))
+            if not isinstance(value, str):
+                raise TypeError("Got {} ({}) for attribute {} from JS: "
+                                "{}".format(value, type(value), name, js_dict))
+        for rect in js_dict['rects']:
+            assert set(rect.keys()) == {'top', 'right', 'bottom', 'left',
+                                        'height', 'width'}, rect.keys()
+            for value in rect.values():
+                if not isinstance(value, (int, float)):
+                    raise TypeError("Got {} ({}) for rect from JS: "
+                                    "{}".format(value, type(value), js_dict))
+
         self._id = js_dict['id']
         self._js_dict = js_dict
 
@@ -88,7 +120,9 @@ class WebEngineElement(webelem.AbstractWebElement):
 
         The returned name will always be lower-case.
         """
-        return self._js_dict['tag_name'].lower()
+        tag = self._js_dict['tag_name']
+        assert isinstance(tag, str), tag
+        return tag.lower()
 
     def outer_xml(self):
         """Get the full HTML representation of this element."""

qutebrowser/browser/webengine/webengineinspector.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webengine/webenginequtescheme.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""QtWebEngine specific qute:* handlers and glue code."""
+"""QtWebEngine specific qute://* handlers and glue code."""
 
 from PyQt5.QtCore import QBuffer, QIODevice
 # pylint: disable=no-name-in-module,import-error,useless-suppression
@@ -26,15 +26,15 @@ from PyQt5.QtWebEngineCore import (QWebEngineUrlSchemeHandler,
 # pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import qutescheme
-from qutebrowser.utils import log
+from qutebrowser.utils import log, qtutils
 
 
 class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
 
-    """Handle qute:* requests on QtWebEngine."""
+    """Handle qute://* requests on QtWebEngine."""
 
     def install(self, profile):
-        """Install the handler for qute: URLs on the given profile."""
+        """Install the handler for qute:// URLs on the given profile."""
         profile.installUrlSchemeHandler(b'qute', self)
 
     def requestStarted(self, job):
@@ -58,12 +58,15 @@ class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
             job.fail(QWebEngineUrlRequestJob.UrlNotFound)
         except qutescheme.QuteSchemeOSError:
             # FIXME:qtwebengine how do we show a better error here?
-            log.misc.exception("OSError while handling qute:* URL")
+            log.misc.exception("OSError while handling qute://* URL")
             job.fail(QWebEngineUrlRequestJob.UrlNotFound)
         except qutescheme.QuteSchemeError:
             # FIXME:qtwebengine how do we show a better error here?
-            log.misc.exception("Error while handling qute:* URL")
+            log.misc.exception("Error while handling qute://* URL")
             job.fail(QWebEngineUrlRequestJob.RequestFailed)
+        except qutescheme.Redirect as e:
+            qtutils.ensure_valid(e.url)
+            job.redirect(e.url)
         else:
             log.misc.debug("Returning {} data".format(mimetype))
 

qutebrowser/browser/webengine/webenginesettings.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -34,7 +34,8 @@ from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile,
 
 from qutebrowser.browser import shared
 from qutebrowser.config import config, websettings
-from qutebrowser.utils import objreg, utils, standarddir, javascript, log
+from qutebrowser.utils import (objreg, utils, standarddir, javascript, log,
+                               qtutils)
 
 
 class Attribute(websettings.Attribute):
@@ -177,6 +178,7 @@ def init(args):
     _init_stylesheet(profile)
     # We need to do this here as a WORKAROUND for
     # https://bugreports.qt.io/browse/QTBUG-58650
+    if not qtutils.version_check('5.9'):
         PersistentCookiePolicy().set(config.get('content', 'cookies-store'))
 
     Attribute(QWebEngineSettings.FullScreenSupportEnabled).set(True)
@@ -221,9 +223,6 @@ MAPPINGS = {
             Attribute(QWebEngineSettings.LocalContentCanAccessRemoteUrls),
         'local-content-can-access-file-urls':
             Attribute(QWebEngineSettings.LocalContentCanAccessFileUrls),
-        # https://bugreports.qt.io/browse/QTBUG-58650
-        # 'cookies-store':
-        #     PersistentCookiePolicy(),
         'webgl':
             Attribute(QWebEngineSettings.WebGLEnabled),
     },
@@ -301,3 +300,8 @@ try:
 except AttributeError:
     # Added in Qt 5.8
     pass
+
+
+if qtutils.version_check('5.9'):
+    # https://bugreports.qt.io/browse/QTBUG-58650
+    MAPPINGS['content']['cookies-store'] = PersistentCookiePolicy()

qutebrowser/browser/webengine/webenginetab.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,9 +17,6 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-# FIXME:qtwebengine remove this once the stubs are gone
-# pylint: disable=unused-variable
-
 """Wrapper over a QWebEngineView."""
 
 import functools
@@ -40,7 +37,7 @@ from qutebrowser.browser.webengine import (webview, webengineelem, tabhistory,
                                            webenginedownloads)
 from qutebrowser.misc import miscwidgets
 from qutebrowser.utils import (usertypes, qtutils, log, javascript, utils,
-                               objreg, jinja)
+                               objreg, jinja, debug)
 
 
 _qute_scheme_handler = None
@@ -55,7 +52,7 @@ def init():
     app = QApplication.instance()
     profile = QWebEngineProfile.defaultProfile()
 
-    log.init.debug("Initializing qute:* handler...")
+    log.init.debug("Initializing qute://* handler...")
     _qute_scheme_handler = webenginequtescheme.QuteSchemeHandler(parent=app)
     _qute_scheme_handler.install(profile)
 
@@ -82,17 +79,17 @@ _JS_WORLD_MAP = {
 
 class WebEngineAction(browsertab.AbstractAction):
 
-    """QtWebKit implementations related to web actions."""
+    """QtWebEngine implementations related to web actions."""
 
-    def _action(self, action):
+    action_class = QWebEnginePage
-        self._widget.triggerPageAction(action)
+    action_base = QWebEnginePage.WebAction
 
     def exit_fullscreen(self):
-        self._action(QWebEnginePage.ExitFullScreen)
+        self._widget.triggerPageAction(QWebEnginePage.ExitFullScreen)
 
     def save_page(self):
         """Save the current page."""
-        self._action(QWebEnginePage.SavePage)
+        self._widget.triggerPageAction(QWebEnginePage.SavePage)
 
 
 class WebEnginePrinting(browsertab.AbstractPrinting):
@@ -128,12 +125,23 @@ class WebEngineSearch(browsertab.AbstractSearch):
         super().__init__(parent)
         self._flags = QWebEnginePage.FindFlags(0)
 
-    def _find(self, text, flags, cb=None):
+    def _find(self, text, flags, callback, caller):
-        """Call findText on the widget with optional callback."""
+        """Call findText on the widget."""
-        if cb is None:
+        self.search_displayed = True
-            self._widget.findText(text, flags)
+
+        def wrapped_callback(found):
+            """Wrap the callback to do debug logging."""
+            found_text = 'found' if found else "didn't find"
+            if flags:
+                flag_text = 'with flags {}'.format(debug.qflags_key(
+                    QWebEnginePage, flags, klass=QWebEnginePage.FindFlag))
             else:
-            self._widget.findText(text, flags, cb)
+                flag_text = ''
+            log.webview.debug(' '.join([caller, found_text, text, flag_text])
+                              .strip())
+            if callback is not None:
+                callback(found)
+        self._widget.findText(text, flags, wrapped_callback)
 
     def search(self, text, *, ignore_case=False, reverse=False,
                result_cb=None):
@@ -148,9 +156,10 @@ class WebEngineSearch(browsertab.AbstractSearch):
 
         self.text = text
         self._flags = flags
-        self._find(text, flags, result_cb)
+        self._find(text, flags, result_cb, 'search')
 
     def clear(self):
+        self.search_displayed = False
         self._widget.findText('')
 
     def prev_result(self, *, result_cb=None):
@@ -160,10 +169,10 @@ class WebEngineSearch(browsertab.AbstractSearch):
             flags &= ~QWebEnginePage.FindBackward
         else:
             flags |= QWebEnginePage.FindBackward
-        self._find(self.text, flags, result_cb)
+        self._find(self.text, flags, result_cb, 'prev_result')
 
     def next_result(self, *, result_cb=None):
-        self._find(self.text, self._flags, result_cb)
+        self._find(self.text, self._flags, result_cb, 'next_result')
 
 
 class WebEngineCaret(browsertab.AbstractCaret):
@@ -237,8 +246,47 @@ class WebEngineCaret(browsertab.AbstractCaret):
             raise browsertab.UnsupportedOperationError
         return self._widget.selectedText()
 
+    def _follow_selected_cb(self, js_elem, tab=False):
+        """Callback for javascript which clicks the selected element.
+
+        Args:
+            js_elem: The element serialized from javascript.
+            tab: Open in a new tab.
+        """
+        if js_elem is None:
+            return
+        assert isinstance(js_elem, dict), js_elem
+        elem = webengineelem.WebEngineElement(js_elem, tab=self._tab)
+        if tab:
+            click_type = usertypes.ClickTarget.tab
+        else:
+            click_type = usertypes.ClickTarget.normal
+
+        # Only click if we see a link
+        if elem.is_link():
+            log.webview.debug("Found link in selection, clicking. ClickTarget "
+                              "{}, elem {}".format(click_type, elem))
+            elem.click(click_type)
+
     def follow_selected(self, *, tab=False):
-        log.stub()
+        if self._tab.search.search_displayed:
+            # We are currently in search mode.
+            # let's click the link via a fake-click
+            # https://bugreports.qt.io/browse/QTBUG-60673
+            self._tab.search.clear()
+
+            log.webview.debug("Clicking a searched link via fake key press.")
+            # send a fake enter, clicking the orange selection box
+            if tab:
+                self._tab.key_press(Qt.Key_Enter, modifier=Qt.ControlModifier)
+            else:
+                self._tab.key_press(Qt.Key_Enter)
+
+        else:
+            # click an existing blue selection
+            js_code = javascript.assemble('webelem', 'find_selected_link')
+            self._tab.run_js_async(js_code, lambda jsret:
+                                   self._follow_selected_cb(jsret, tab))
 
 
 class WebEngineScroller(browsertab.AbstractScroller):
@@ -256,13 +304,10 @@ class WebEngineScroller(browsertab.AbstractScroller):
         page = widget.page()
         page.scrollPositionChanged.connect(self._update_pos)
 
-    def _key_press(self, key, count=1):
+    def _repeated_key_press(self, key, count=1, modifier=Qt.NoModifier):
+        """Send count fake key presses to this scroller's WebEngineTab."""
         for _ in range(min(count, 5000)):
-            press_evt = QKeyEvent(QEvent.KeyPress, key, Qt.NoModifier, 0, 0, 0)
+            self._tab.key_press(key, modifier)
-            release_evt = QKeyEvent(QEvent.KeyRelease, key, Qt.NoModifier,
-                                    0, 0, 0)
-            self._tab.send_event(press_evt)
-            self._tab.send_event(release_evt)
 
     @pyqtSlot()
     def _update_pos(self):
@@ -319,28 +364,28 @@ class WebEngineScroller(browsertab.AbstractScroller):
         self._tab.run_js_async(js_code)
 
     def up(self, count=1):
-        self._key_press(Qt.Key_Up, count)
+        self._repeated_key_press(Qt.Key_Up, count)
 
     def down(self, count=1):
-        self._key_press(Qt.Key_Down, count)
+        self._repeated_key_press(Qt.Key_Down, count)
 
     def left(self, count=1):
-        self._key_press(Qt.Key_Left, count)
+        self._repeated_key_press(Qt.Key_Left, count)
 
     def right(self, count=1):
-        self._key_press(Qt.Key_Right, count)
+        self._repeated_key_press(Qt.Key_Right, count)
 
     def top(self):
-        self._key_press(Qt.Key_Home)
+        self._tab.key_press(Qt.Key_Home)
 
     def bottom(self):
-        self._key_press(Qt.Key_End)
+        self._tab.key_press(Qt.Key_End)
 
     def page_up(self, count=1):
-        self._key_press(Qt.Key_PageUp, count)
+        self._repeated_key_press(Qt.Key_PageUp, count)
 
     def page_down(self, count=1):
-        self._key_press(Qt.Key_PageDown, count)
+        self._repeated_key_press(Qt.Key_PageDown, count)
 
     def at_top(self):
         return self.pos_px().y() == 0
@@ -369,9 +414,13 @@ class WebEngineHistory(browsertab.AbstractHistory):
         return self._history.canGoForward()
 
     def serialize(self):
-        # WORKAROUND for https://github.com/qutebrowser/qutebrowser/issues/2289
+        if not qtutils.version_check('5.9'):
-        # FIXME:qtwebengine can we get rid of this with Qt 5.8.1?
+            # WORKAROUND for
-        scheme = self._history.currentItem().url().scheme()
+            # https://github.com/qutebrowser/qutebrowser/issues/2289
+            # Don't use the history's currentItem here, because of
+            # https://bugreports.qt.io/browse/QTBUG-59599 and because it doesn't
+            # contain view-source.
+            scheme = self._tab.url().scheme()
             if scheme in ['view-source', 'chrome']:
                 raise browsertab.WebTabError("Can't serialize special URL!")
         return qtutils.serialize(self._history)
@@ -596,6 +645,13 @@ class WebEngineTab(browsertab.AbstractTab):
     def clear_ssl_errors(self):
         raise browsertab.UnsupportedOperationError
 
+    def key_press(self, key, modifier=Qt.NoModifier):
+        press_evt = QKeyEvent(QEvent.KeyPress, key, modifier, 0, 0, 0)
+        release_evt = QKeyEvent(QEvent.KeyRelease, key, modifier,
+                                0, 0, 0)
+        self.send_event(press_evt)
+        self.send_event(release_evt)
+
     @pyqtSlot()
     def _on_history_trigger(self):
         url = self.url()

qutebrowser/browser/webengine/webview.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/cache.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -25,7 +25,7 @@ from PyQt5.QtCore import pyqtSlot
 from PyQt5.QtNetwork import QNetworkDiskCache, QNetworkCacheMetaData
 
 from qutebrowser.config import config
-from qutebrowser.utils import utils, objreg
+from qutebrowser.utils import utils, objreg, qtutils
 
 
 class DiskCache(QNetworkDiskCache):
@@ -53,6 +53,10 @@ class DiskCache(QNetworkDiskCache):
         size = config.get('storage', 'cache-size')
         if size is None:
             size = 1024 * 1024 * 50  # default from QNetworkDiskCachePrivate
+        # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-59909
+        if (qtutils.version_check('5.7.1') and
+                not qtutils.version_check('5.9')):  # pragma: no cover
+            size = 0
         self.setMaximumCacheSize(size)
 
     def _maybe_activate(self):

qutebrowser/browser/webkit/certificateerror.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -41,7 +41,7 @@ class CertificateErrorWrapper(usertypes.AbstractCertificateErrorWrapper):
         try:
             # Qt >= 5.4
             return hash(self._error)
-        except TypeError:
+        except TypeError:  # pragma: no cover
             return hash((self._error.certificate().toDer(),
                          self._error.error()))
 

qutebrowser/browser/webkit/cookies.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/http.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/mhtml.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Daniel Schadt
+# Copyright 2015-2017 Daniel Schadt
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/network/filescheme.py

@@ -1,7 +1,7 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-# Copyright 2015-2016 Antoni Boucher (antoyo) <bouanto@zoho.com>
+# Copyright 2015-2017 Antoni Boucher (antoyo) <bouanto@zoho.com>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/network/networkmanager.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -416,8 +416,7 @@ class NetworkManager(QNetworkAccessManager):
             req.setRawHeader(header, value)
 
         host_blocker = objreg.get('host-blocker')
-        if (op == QNetworkAccessManager.GetOperation and
+        if host_blocker.is_blocked(req.url()):
-                host_blocker.is_blocked(req.url())):
             log.webview.info("Request to {} blocked by host blocker.".format(
                 req.url().host()))
             return networkreply.ErrorNetworkReply(

qutebrowser/browser/webkit/network/networkreply.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # Based on the Eric5 helpviewer,
 # Copyright (c) 2009 - 2014 Detlev Offenbach <detlev@die-offenbachs.de>
@@ -19,6 +19,10 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+#
+# For some reason, a segfault will be triggered if the unnecessary lambdas in
+# this file aren't there.
+# pylint: disable=unnecessary-lambda
 
 """Special network replies.."""
 
@@ -114,9 +118,6 @@ class ErrorNetworkReply(QNetworkReply):
         # the device to avoid getting a warning.
         self.setOpenMode(QIODevice.ReadOnly)
         self.setError(error, errorstring)
-        # For some reason, a segfault will be triggered if these lambdas aren't
-        # there.
-        # pylint: disable=unnecessary-lambda
         QTimer.singleShot(0, lambda: self.error.emit(error))
         QTimer.singleShot(0, lambda: self.finished.emit())
 
@@ -137,3 +138,20 @@ class ErrorNetworkReply(QNetworkReply):
 
     def isRunning(self):
         return False
+
+
+class RedirectNetworkReply(QNetworkReply):
+
+    """A reply which redirects to the given URL."""
+
+    def __init__(self, new_url, parent=None):
+        super().__init__(parent)
+        self.setAttribute(QNetworkRequest.RedirectionTargetAttribute, new_url)
+        QTimer.singleShot(0, lambda: self.finished.emit())
+
+    def abort(self):
+        """Called when there's e.g. a redirection limit."""
+        pass
+
+    def readData(self, _maxlen):
+        return bytes()

qutebrowser/browser/webkit/network/schemehandler.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # Based on the Eric5 helpviewer,
 # Copyright (c) 2009 - 2014 Detlev Offenbach <detlev@die-offenbachs.de>

qutebrowser/browser/webkit/network/webkitqutescheme.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""QtWebKit specific qute:* handlers and glue code."""
+"""QtWebKit specific qute://* handlers and glue code."""
 
 import mimetypes
 import functools
@@ -28,13 +28,13 @@ from PyQt5.QtNetwork import QNetworkReply
 
 from qutebrowser.browser import pdfjs, qutescheme
 from qutebrowser.browser.webkit.network import schemehandler, networkreply
-from qutebrowser.utils import jinja, log, message, objreg, usertypes
+from qutebrowser.utils import jinja, log, message, objreg, usertypes, qtutils
 from qutebrowser.config import configexc, configdata
 
 
 class QuteSchemeHandler(schemehandler.SchemeHandler):
 
-    """Scheme handler for qute: URLs."""
+    """Scheme handler for qute:// URLs."""
 
     def createRequest(self, _op, request, _outgoing_data):
         """Create a new request.
@@ -62,6 +62,9 @@ class QuteSchemeHandler(schemehandler.SchemeHandler):
         except qutescheme.QuteSchemeError as e:
             return networkreply.ErrorNetworkReply(request, e.errorstring,
                                                   e.error, self.parent())
+        except qutescheme.Redirect as e:
+            qtutils.ensure_valid(e.url)
+            return networkreply.RedirectNetworkReply(e.url, self.parent())
 
         return networkreply.FixedDataNetworkReply(request, data, mimetype,
                                                   self.parent())
@@ -69,15 +72,15 @@ class QuteSchemeHandler(schemehandler.SchemeHandler):
 
 class JSBridge(QObject):
 
-    """Javascript-bridge for special qute:... pages."""
+    """Javascript-bridge for special qute://... pages."""
 
     @pyqtSlot(str, str, str)
     def set(self, sectname, optname, value):
-        """Slot to set a setting from qute:settings."""
+        """Slot to set a setting from qute://settings."""
         # https://github.com/qutebrowser/qutebrowser/issues/727
         if ((sectname, optname) == ('content', 'allow-javascript') and
                 value == 'false'):
-            message.error("Refusing to disable javascript via qute:settings "
+            message.error("Refusing to disable javascript via qute://settings "
                           "as it needs javascript support.")
             return
         try:
@@ -88,7 +91,7 @@ class JSBridge(QObject):
 
 @qutescheme.add_handler('settings', backend=usertypes.Backend.QtWebKit)
 def qute_settings(_url):
-    """Handler for qute:settings. View/change qute configuration."""
+    """Handler for qute://settings. View/change qute configuration."""
     config_getter = functools.partial(objreg.get('config').get, raw=True)
     html = jinja.render('settings.html', title='settings', config=configdata,
                         confget=config_getter)

qutebrowser/browser/webkit/rfc6266.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -286,9 +286,6 @@ def normalize_ws(text):
 
 def parse_headers(content_disposition):
     """Build a _ContentDisposition from header values."""
-    # WORKAROUND for https://bitbucket.org/logilab/pylint/issue/492/
-    # pylint: disable=no-member
-
     # We allow non-ascii here (it will only be parsed inside of qdtext, and
     # rejected by the grammar if it appears in other places), although parsing
     # it can be ambiguous.  Parsing it ensures that a non-ambiguous filename*

qutebrowser/browser/webkit/tabhistory.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -21,7 +21,6 @@
 
 
 from PyQt5.QtCore import QByteArray, QDataStream, QIODevice, QUrl
-from PyQt5.QtWebKit import qWebKitVersion
 
 from qutebrowser.utils import qtutils
 
@@ -181,7 +180,7 @@ def serialize(items):
     else:
         current_idx = 0
 
-    if qtutils.is_qtwebkit_ng(qWebKitVersion()):
+    if qtutils.is_qtwebkit_ng():
         _serialize_ng(items, current_idx, stream)
     else:
         _serialize_old(items, current_idx, stream)

qutebrowser/browser/webkit/webkitelem.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -112,7 +112,9 @@ class WebKitElement(webelem.AbstractWebElement):
 
     def value(self):
         self._check_vanished()
-        return self._elem.evaluateJavaScript('this.value')
+        val = self._elem.evaluateJavaScript('this.value')
+        assert isinstance(val, (int, float, str)), val
+        return val
 
     def set_value(self, value):
         self._check_vanished()
@@ -283,8 +285,7 @@ class WebKitElement(webelem.AbstractWebElement):
         for _ in range(5):
             if elem is None:
                 break
-            tag = elem.tag_name()
+            if elem.is_link():
-            if tag == 'a' or tag == 'area':
                 if elem.get('target', None) == '_blank':
                     elem['target'] = '_top'
                 break

qutebrowser/browser/webkit/webkithistory.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/webkitinspector.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/webkitsettings.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -26,7 +26,7 @@ Module attributes:
 
 import os.path
 
-from PyQt5.QtWebKit import QWebSettings, qWebKitVersion
+from PyQt5.QtWebKit import QWebSettings
 
 from qutebrowser.config import config, websettings
 from qutebrowser.utils import standarddir, objreg, urlutils, qtutils, message
@@ -90,7 +90,7 @@ def _set_user_stylesheet():
 
 def _init_private_browsing():
     if config.get('general', 'private-browsing'):
-        if qtutils.is_qtwebkit_ng(qWebKitVersion()):
+        if qtutils.is_qtwebkit_ng():
             message.warning("Private browsing is not fully implemented by "
                             "QtWebKit-NG!")
         QWebSettings.setIconDatabasePath('')

qutebrowser/browser/webkit/webkittab.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -23,6 +23,7 @@ import sys
 import functools
 import xml.etree.ElementTree
 
+import sip
 from PyQt5.QtCore import (pyqtSlot, Qt, QEvent, QUrl, QPoint, QTimer, QSizeF,
                           QSize)
 from PyQt5.QtGui import QKeyEvent
@@ -35,7 +36,7 @@ from qutebrowser.browser import browsertab
 from qutebrowser.browser.network import proxy
 from qutebrowser.browser.webkit import webview, tabhistory, webkitelem
 from qutebrowser.browser.webkit.network import webkitqutescheme
-from qutebrowser.utils import qtutils, objreg, usertypes, utils, log
+from qutebrowser.utils import qtutils, objreg, usertypes, utils, log, debug
 
 
 def init():
@@ -56,6 +57,9 @@ class WebKitAction(browsertab.AbstractAction):
 
     """QtWebKit implementations related to web actions."""
 
+    action_class = QWebPage
+    action_base = QWebPage.WebAction
+
     def exit_fullscreen(self):
         raise browsertab.UnsupportedOperationError
 
@@ -103,7 +107,7 @@ class WebKitSearch(browsertab.AbstractSearch):
         super().__init__(parent)
         self._flags = QWebPage.FindFlags(0)
 
-    def _call_cb(self, callback, found):
+    def _call_cb(self, callback, found, text, flags, caller):
         """Call the given callback if it's non-None.
 
         Delays the call via a QTimer so the website is re-rendered in between.
@@ -111,17 +115,34 @@ class WebKitSearch(browsertab.AbstractSearch):
         Args:
             callback: What to call
             found: If the text was found
+            text: The text searched for
+            flags: The flags searched with
+            caller: Name of the caller.
         """
+        found_text = 'found' if found else "didn't find"
+        # Removing FindWrapsAroundDocument to get the same logging as with
+        # QtWebEngine
+        debug_flags = debug.qflags_key(
+            QWebPage, flags & ~QWebPage.FindWrapsAroundDocument,
+            klass=QWebPage.FindFlag)
+        if debug_flags != '0x0000':
+            flag_text = 'with flags {}'.format(debug_flags)
+        else:
+            flag_text = ''
+        log.webview.debug(' '.join([caller, found_text, text, flag_text])
+                          .strip())
         if callback is not None:
             QTimer.singleShot(0, functools.partial(callback, found))
 
     def clear(self):
+        self.search_displayed = False
         # We first clear the marked text, then the highlights
         self._widget.findText('')
         self._widget.findText('', QWebPage.HighlightAllOccurrences)
 
     def search(self, text, *, ignore_case=False, reverse=False,
                result_cb=None):
+        self.search_displayed = True
         flags = QWebPage.FindWrapsAroundDocument
         if ignore_case == 'smart':
             if not text.islower():
@@ -136,13 +157,15 @@ class WebKitSearch(browsertab.AbstractSearch):
         self._widget.findText(text, flags | QWebPage.HighlightAllOccurrences)
         self.text = text
         self._flags = flags
-        self._call_cb(result_cb, found)
+        self._call_cb(result_cb, found, text, flags, 'search')
 
     def next_result(self, *, result_cb=None):
+        self.search_displayed = True
         found = self._widget.findText(self.text, self._flags)
-        self._call_cb(result_cb, found)
+        self._call_cb(result_cb, found, self.text, self._flags, 'next_result')
 
     def prev_result(self, *, result_cb=None):
+        self.search_displayed = True
         # The int() here makes sure we get a copy of the flags.
         flags = QWebPage.FindFlags(int(self._flags))
         if flags & QWebPage.FindBackward:
@@ -150,7 +173,7 @@ class WebKitSearch(browsertab.AbstractSearch):
         else:
             flags |= QWebPage.FindBackward
         found = self._widget.findText(self.text, flags)
-        self._call_cb(result_cb, found)
+        self._call_cb(result_cb, found, self.text, flags, 'prev_result')
 
 
 class WebKitCaret(browsertab.AbstractCaret):
@@ -444,15 +467,11 @@ class WebKitScroller(browsertab.AbstractScroller):
         # self._widget.setFocus()
 
         for _ in range(min(count, 5000)):
-            press_evt = QKeyEvent(QEvent.KeyPress, key, Qt.NoModifier, 0, 0, 0)
-            release_evt = QKeyEvent(QEvent.KeyRelease, key, Qt.NoModifier,
-                                    0, 0, 0)
             # Abort scrolling if the minimum/maximum was reached.
             if (getter is not None and
                     frame.scrollBarValue(direction) == getter(direction)):
                 return
-            self._widget.keyPressEvent(press_evt)
+            self._tab.key_press(key)
-            self._widget.keyReleaseEvent(release_evt)
 
     def up(self, count=1):
         self._key_press(Qt.Key_Up, count, 'scrollBarMinimum', Qt.Vertical)
@@ -678,6 +697,13 @@ class WebKitTab(browsertab.AbstractTab):
     def clear_ssl_errors(self):
         self.networkaccessmanager().clear_all_ssl_errors()
 
+    def key_press(self, key, modifier=Qt.NoModifier):
+        press_evt = QKeyEvent(QEvent.KeyPress, key, modifier, 0, 0, 0)
+        release_evt = QKeyEvent(QEvent.KeyRelease, key, modifier,
+                                0, 0, 0)
+        self.send_event(press_evt)
+        self.send_event(release_evt)
+
     @pyqtSlot()
     def _on_history_trigger(self):
         url = self.url()
@@ -707,6 +733,9 @@ class WebKitTab(browsertab.AbstractTab):
     @pyqtSlot()
     def _on_webkit_icon_changed(self):
         """Emit iconChanged with a QIcon like QWebEngineView does."""
+        if sip.isdeleted(self._widget):
+            log.webview.debug("Got _on_webkit_icon_changed for deleted view!")
+            return
         self.icon_changed.emit(self._widget.icon())
 
     @pyqtSlot(QWebFrame)

qutebrowser/browser/webkit/webpage.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/webview.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -140,7 +140,7 @@ class WebView(QWebView):
 
     @pyqtSlot()
     def add_js_bridge(self):
-        """Add the javascript bridge for qute:... pages."""
+        """Add the javascript bridge for qute://... pages."""
         frame = self.sender()
         if not isinstance(frame, QWebFrame):
             log.webview.error("Got non-QWebFrame {!r} in "

qutebrowser/commands/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/commands/argparser.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -76,11 +76,11 @@ class ArgumentParser(argparse.ArgumentParser):
         self.name = name
         super().__init__(*args, add_help=False, prog=name, **kwargs)
 
-    def exit(self, status=0, msg=None):
+    def exit(self, status=0, message=None):
-        raise ArgumentParserExit(status, msg)
+        raise ArgumentParserExit(status, message)
 
-    def error(self, msg):
+    def error(self, message):
-        raise ArgumentParserError(msg.capitalize())
+        raise ArgumentParserError(message.capitalize())
 
 
 def arg_name(name):

qutebrowser/commands/cmdexc.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/commands/cmdutils.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/commands/command.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/commands/runners.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -133,7 +133,8 @@ class CommandRunner(QObject):
         Yields:
             ParseResult tuples.
         """
-        if not text.strip():
+        text = text.strip().lstrip(':').strip()
+        if not text:
             raise cmdexc.NoSuchCommandError("No command given")
 
         if aliases:

qutebrowser/commands/userscripts.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/completion/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/completion/completer.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/completion/completiondelegate.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/completion/completionwidget.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -23,7 +23,7 @@ Defines a CompletionView which uses CompletionFiterModel and CompletionModel
 subclasses to provide completions.
 """
 
-from PyQt5.QtWidgets import QStyle, QTreeView, QSizePolicy
+from PyQt5.QtWidgets import QStyle, QTreeView, QSizePolicy, QStyleFactory
 from PyQt5.QtCore import pyqtSlot, pyqtSignal, Qt, QItemSelectionModel, QSize
 
 from qutebrowser.config import config, style
@@ -117,6 +117,7 @@ class CompletionView(QTreeView):
 
         self._delegate = completiondelegate.CompletionItemDelegate(self)
         self.setItemDelegate(self._delegate)
+        self.setStyle(QStyleFactory.create('Fusion'))
         style.set_register_stylesheet(self)
         self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)
         self.setHeaderHidden(True)
@@ -124,6 +125,7 @@ class CompletionView(QTreeView):
         self.setIndentation(0)
         self.setItemsExpandable(False)
         self.setExpandsOnDoubleClick(False)
+        self.setAnimated(False)
         self.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
         # WORKAROUND
         # This is a workaround for weird race conditions with invalid

qutebrowser/completion/models/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/completion/models/base.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -103,7 +103,7 @@ class BaseCompletionModel(QStandardItemModel):
             nameitem.setData(userdata, Role.userdata)
         return nameitem, descitem, miscitem
 
-    def delete_cur_item(self, win_id):
+    def delete_cur_item(self, completion):
         """Delete the selected item."""
         raise NotImplementedError