Merge remote-tracking branch 'upstream/master' into regex-match

doc/changelog.asciidoc

@@ -57,6 +57,7 @@ Added
   when restoring a session.
 - New `hist_importer.py` script to import history from Firefox/Chromium.
 - New `{protocol}` replacement for `tabs.title.format` and friends.
+- New `-o` flag for `:spawn` to show stdout/stderr in a new tab.
 
 Changed
 ~~~~~~~
@@ -118,6 +119,7 @@ Fixed
 - Fixed crash when closing the tab an external editor was opened in.
 - When using `:search-next` before a search is finished, no warning about no
   results being found is shown anymore.
+- Fix :click-element with an ID containing non-alphanumeric characters.
 
 Deprecated
 ~~~~~~~~~~

doc/help/commands.asciidoc

@@ -1148,7 +1148,7 @@ Set a mark at the current scroll position in the current tab.
 
 [[spawn]]
 === spawn
-Syntax: +:spawn [*--userscript*] [*--verbose*] [*--detach*] 'cmdline'+
+Syntax: +:spawn [*--userscript*] [*--verbose*] [*--output*] [*--detach*] 'cmdline'+
 
 Spawn a command in a shell.
 
@@ -1163,6 +1163,7 @@ Spawn a command in a shell.
  - `/usr/share/qutebrowser/userscripts`
 
 * +*-v*+, +*--verbose*+: Show notifications when the command started/exited.
+* +*-o*+, +*--output*+: Whether the output should be shown in a new tab.
 * +*-d*+, +*--detach*+: Whether the command should be detached from qutebrowser.
 
 ==== note

misc/requirements/requirements-codecov.txt

@@ -2,7 +2,7 @@
 
 certifi==2017.11.5
 chardet==3.0.4
-codecov==2.0.9
+codecov==2.0.10
 coverage==4.4.2
 idna==2.6
 requests==2.18.4

misc/requirements/requirements-pip.txt

@@ -3,6 +3,6 @@
 appdirs==1.4.3
 packaging==16.8
 pyparsing==2.2.0
-setuptools==38.2.3
+setuptools==38.2.4
 six==1.11.0
 wheel==0.30.0

misc/requirements/requirements-pylint.txt

@@ -8,7 +8,7 @@ idna==2.6
 isort==4.2.15
 lazy-object-proxy==1.3.1
 mccabe==0.6.1
-pylint==1.7.4
+pylint==1.7.5
 ./scripts/dev/pylint_checkers
 requests==2.18.4
 six==1.11.0

misc/requirements/requirements-tests.txt

@@ -2,7 +2,7 @@
 
 attrs==17.3.0
 beautifulsoup4==4.6.0
-cheroot==5.10.0
+cheroot==6.0.0
 click==6.7
 # colorama==0.3.9
 coverage==4.4.2
@@ -11,7 +11,7 @@ fields==5.0.0
 Flask==0.12.2
 glob2==0.6
 hunter==2.0.2
-hypothesis==3.40.1
+hypothesis==3.42.1
 itsdangerous==0.24
 # Jinja2==2.10
 Mako==1.0.7
@@ -21,7 +21,7 @@ parse-type==0.4.2
 pluggy==0.6.0
 py==1.5.2
 py-cpuinfo==3.3.0
-pytest==3.3.0
+pytest==3.3.1
 pytest-bdd==2.19.0
 pytest-benchmark==3.1.1
 pytest-cov==2.5.1
@@ -36,4 +36,4 @@ pytest-xvfb==1.0.0
 PyVirtualDisplay==0.2.1
 six==1.11.0
 vulture==0.26
-Werkzeug==0.12.2
+Werkzeug==0.13

qutebrowser/browser/commands.py

@@ -1177,7 +1177,8 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0, no_replace_variables=True)
-    def spawn(self, cmdline, userscript=False, verbose=False, detach=False):
+    def spawn(self, cmdline, userscript=False, verbose=False,
+              output=False, detach=False):
         """Spawn a command in a shell.
 
         Args:
@@ -1188,6 +1189,7 @@ class CommandDispatcher:
                               (or `$XDG_DATA_DIR`)
                             - `/usr/share/qutebrowser/userscripts`
             verbose: Show notifications when the command started/exited.
+            output: Whether the output should be shown in a new tab.
             detach: Whether the command should be detached from qutebrowser.
             cmdline: The commandline to execute.
         """
@@ -1214,6 +1216,11 @@ class CommandDispatcher:
             else:
                 proc.start(cmd, args)
 
+        if output:
+            tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                        window='last-focused')
+            tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True)
+
     @cmdutils.register(instance='command-dispatcher', scope='window')
     def home(self):
         """Open main startpage in current tab."""

qutebrowser/browser/qutescheme.py

@@ -42,6 +42,7 @@ from qutebrowser.misc import objects
 
 
 pyeval_output = ":pyeval was never called"
+spawn_output = ":spawn was never called"
 
 
 _HANDLERS = {}
@@ -268,6 +269,13 @@ def qute_pyeval(_url):
     return 'text/html', html
 
 
+@add_handler('spawn-output')
+def qute_spawn_output(_url):
+    """Handler for qute://spawn-output."""
+    html = jinja.render('pre.html', title='spawn output', content=spawn_output)
+    return 'text/html', html
+
+
 @add_handler('version')
 @add_handler('verizon')
 def qute_version(_url):

qutebrowser/browser/webkit/webkittab.py

@@ -19,6 +19,7 @@
 
 """Wrapper over our (QtWebKit) WebView."""
 
+import re
 import functools
 import xml.etree.ElementTree
 
@@ -545,6 +546,10 @@ class WebKitElements(browsertab.AbstractElements):
                 callback(None)
             else:
                 callback(elems[0])
+
+        # Escape non-alphanumeric characters in the selector
+        # https://www.w3.org/TR/CSS2/syndata.html#value-def-identifier
+        elem_id = re.sub(r'[^a-zA-Z0-9_-]', r'\\\g<0>', elem_id)
         self.find_css('#' + elem_id, find_id_cb)
 
     def find_focused(self, callback):

qutebrowser/misc/guiprocess.py

@@ -25,6 +25,7 @@ from PyQt5.QtCore import (pyqtSlot, pyqtSignal, QObject, QProcess,
                           QProcessEnvironment)
 
 from qutebrowser.utils import message, log
+from qutebrowser.browser import qutescheme
 
 # A mapping of QProcess::ErrorCode's to human-readable strings.
 
@@ -96,6 +97,13 @@ class GUIProcess(QObject):
         self._started = False
         log.procs.debug("Process finished with code {}, status {}.".format(
             code, status))
+
+        stderr = bytes(self._proc.readAllStandardError()).decode('utf-8')
+        stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8')
+
+        qutescheme.spawn_output = self._spawn_format(code, status,
+                                                     stdout, stderr)
+
         if status == QProcess.CrashExit:
             message.error("{} crashed!".format(self._what.capitalize()))
         elif status == QProcess.NormalExit and code == 0:
@@ -109,13 +117,22 @@ class GUIProcess(QObject):
             message.error("{} exited with status {}, see :messages for "
                           "details.".format(self._what.capitalize(), code))
 
-            stderr = bytes(self._proc.readAllStandardError()).decode('utf-8')
-            stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8')
             if stdout:
                 log.procs.error("Process stdout:\n" + stdout.strip())
             if stderr:
                 log.procs.error("Process stderr:\n" + stderr.strip())
 
+    def _spawn_format(self, code=0, status=0, stdout="", stderr=""):
+        """Produce a formatted string for spawn output."""
+        stdout = (stdout or "(No output)").strip()
+        stderr = (stderr or "(No output)").strip()
+
+        spawn_string = ("Process finished with code {}, status {}\n"
+                        "\nProcess stdout:\n {}"
+                        "\nProcess stderr:\n {}").format(code, status,
+                                                         stdout, stderr)
+        return spawn_string
+
     @pyqtSlot()
     def on_started(self):
         """Called when the process started successfully."""

scripts/dev/ci/travis_run.sh

@@ -26,7 +26,7 @@ elif [[ $TESTENV == shellcheck ]]; then
            koalaman/shellcheck:latest "${scripts[@]}"
 else
     args=()
-    [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb')
+    [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb' 'tests/unit')
 
     tox -e "$TESTENV" -- "${args[@]}"
 fi

scripts/dev/run_vulture.py

@@ -82,6 +82,7 @@ def whitelist_generator():  # noqa
     yield 'qutebrowser.utils.jinja.Loader.get_source'
     yield 'qutebrowser.utils.log.QtWarningFilter.filter'
     yield 'qutebrowser.browser.pdfjs.is_available'
+    yield 'qutebrowser.misc.guiprocess.spawn_output'
     yield 'QEvent.posted'
     yield 'log_stack'  # from message.py
     yield 'propagate'  # logging.getLogger('...).propagate = False

tests/end2end/data/click_element.html

@@ -9,5 +9,6 @@
         <span>Duplicate</span>
         <form><input id='qute-input'></input></form>
         <a href="/data/hello.txt" id='link'>link</a>
+        <span id='foo.bar' onclick='console.log("id with dot")'>ID with dot</span>
     </body>
 </html>

tests/end2end/features/misc.feature

@@ -449,6 +449,11 @@ Feature: Various utility commands.
         And I run :click-element id qute-input
         Then "Entering mode KeyMode.insert (reason: clicking input)" should be logged
 
+    Scenario: Clicking an element by ID with dot
+        When I open data/click_element.html
+        And I run :click-element id foo.bar
+        Then the javascript message "id with dot" should be logged
+
     Scenario: Clicking an element with tab target
         When I open data/click_element.html
         And I run :tab-only

tests/end2end/features/test_editor_bdd.py

@@ -26,6 +26,8 @@ import signal
 import pytest_bdd as bdd
 bdd.scenarios('editor.feature')
 
+from qutebrowser.utils import utils
+
 
 @bdd.when(bdd.parsers.parse('I set up a fake editor replacing "{text}" by '
                             '"{replacement}"'))
@@ -71,6 +73,7 @@ def set_up_editor_empty(quteproc, tmpdir):
 @bdd.when(bdd.parsers.parse('I set up a fake editor that waits'))
 def set_up_editor_wait(quteproc, tmpdir):
     """Set up editor.command to a small python script inserting a text."""
+    assert not utils.is_windows
     pidfile = tmpdir / 'editor_pid'
     script = tmpdir / 'script.py'
     script.write(textwrap.dedent("""

tests/end2end/fixtures/testprocess.py

@@ -300,8 +300,16 @@ class Process(QObject):
 
     def terminate(self):
         """Clean up and shut down the process."""
+        if not self.is_running():
+            return
+
+        if quteutils.is_windows:
+            self.proc.kill()
+        else:
             self.proc.terminate()
-        self.proc.waitForFinished()
+
+        ok = self.proc.waitForFinished()
+        assert ok
 
     def is_running(self):
         """Check if the process is currently running."""

tests/end2end/fixtures/webserver.py

@@ -172,11 +172,6 @@ class WebserverProcess(testprocess.Process):
     def _default_args(self):
         return [str(self.port)]
 
-    def cleanup(self):
-        """Clean up and shut down the process."""
-        self.proc.terminate()
-        self.proc.waitForFinished()
-
 
 @pytest.fixture(scope='session', autouse=True)
 def server(qapp):
@@ -184,7 +179,7 @@ def server(qapp):
     server = WebserverProcess('webserver_sub')
     server.start()
     yield server
-    server.cleanup()
+    server.terminate()
 
 
 @pytest.fixture(autouse=True)
@@ -208,4 +203,4 @@ def ssl_server(request, qapp):
     server.start()
     yield server
     server.after_test()
-    server.cleanup()
+    server.terminate()

tests/unit/misc/test_guiprocess.py

@@ -19,7 +19,6 @@
 
 """Tests for qutebrowser.misc.guiprocess."""
 
-import json
 import logging
 
 import pytest
@@ -27,6 +26,7 @@ from PyQt5.QtCore import QProcess, QIODevice
 
 from qutebrowser.misc import guiprocess
 from qutebrowser.utils import usertypes
+from qutebrowser.browser import qutescheme
 
 
 @pytest.fixture()
@@ -60,7 +60,7 @@ def test_start(proc, qtbot, message_mock, py_proc):
         proc.start(*argv)
 
     assert not message_mock.messages
-    assert bytes(proc._proc.readAll()).rstrip() == b'test'
+    assert qutescheme.spawn_output == proc._spawn_format(stdout="test")
 
 
 def test_start_verbose(proc, qtbot, message_mock, py_proc):
@@ -77,7 +77,7 @@ def test_start_verbose(proc, qtbot, message_mock, py_proc):
     assert msgs[1].level == usertypes.MessageLevel.info
     assert msgs[0].text.startswith("Executing:")
     assert msgs[1].text == "Testprocess exited successfully."
-    assert bytes(proc._proc.readAll()).rstrip() == b'test'
+    assert qutescheme.spawn_output == proc._spawn_format(stdout="test")
 
 
 def test_start_env(monkeypatch, qtbot, py_proc):
@@ -99,10 +99,9 @@ def test_start_env(monkeypatch, qtbot, py_proc):
                            order='strict'):
         proc.start(*argv)
 
-    data = bytes(proc._proc.readAll()).decode('utf-8')
+    data = qutescheme.spawn_output
-    ret_env = json.loads(data)
+    assert 'QUTEBROWSER_TEST_1' in data
-    assert 'QUTEBROWSER_TEST_1' in ret_env
+    assert 'QUTEBROWSER_TEST_2' in data
-    assert 'QUTEBROWSER_TEST_2' in ret_env
 
 
 @pytest.mark.qt_log_ignore('QIODevice::read.*: WriteOnly device')