From 568d60753e4856e9f84921e1084ea23d4e9e5956 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andor=20Uhl=C3=A1r?= <neferty@neferty.me>
Date: Sun, 10 May 2015 18:33:36 +0200
Subject: [PATCH 01/35] Add greasemonkey compatible userscript module.

WebKit backend only for now. Loads all .js files from a directory,
specified in the greasemonkey-directory key in the storage section,
defaulting to data/greasemonkey, and wraps them in a minimal environment
providing some GM_* functions. Makes those scripts available via the
"greasemonkey" registered object in objreg and injects scripts at appropriate
times in a page load base on @run-at directives.
---
 qutebrowser/browser/browsertab.py     |   6 +-
 qutebrowser/browser/greasemonkey.py   | 262 ++++++++++++++++++++++++++
 qutebrowser/browser/webkit/webpage.py |  28 +++
 qutebrowser/utils/log.py              |   4 +-
 4 files changed, 298 insertions(+), 2 deletions(-)
 create mode 100644 qutebrowser/browser/greasemonkey.py

diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py
index 547e276db..866943f87 100644
--- a/qutebrowser/browser/browsertab.py
+++ b/qutebrowser/browser/browsertab.py
@@ -31,7 +31,7 @@ from qutebrowser.keyinput import modeman
 from qutebrowser.config import config
 from qutebrowser.utils import utils, objreg, usertypes, log, qtutils
 from qutebrowser.misc import miscwidgets, objects
-from qutebrowser.browser import mouse, hints
+from qutebrowser.browser import mouse, hints, greasemonkey
 
 
 tab_id_gen = itertools.count(0)
@@ -64,6 +64,10 @@ def init():
         from qutebrowser.browser.webengine import webenginetab
         webenginetab.init()
 
+    log.init.debug("Initializing Greasemonkey...")
+    gm_manager = greasemonkey.GreasemonkeyManager()
+    objreg.register('greasemonkey', gm_manager)
+
 
 class WebTabError(Exception):
 
diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
new file mode 100644
index 000000000..d1f63a4a4
--- /dev/null
+++ b/qutebrowser/browser/greasemonkey.py
@@ -0,0 +1,262 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Load, parse and make avalaible greasemonkey scripts."""
+
+import re
+import os
+import json
+import fnmatch
+import functools
+import glob
+
+from PyQt5.QtCore import pyqtSignal, QObject
+
+from qutebrowser.utils import log, standarddir
+
+# TODO: GM_ bootstrap
+
+
+def _scripts_dir():
+    """Get the directory of the scripts."""
+    return os.path.join(standarddir.data(), 'greasemonkey')
+
+
+class GreasemonkeyScript:
+    """Container class for userscripts, parses metadata blocks."""
+
+    GM_BOOTSTRAP_TEMPLATE = r"""var _qute_script_id = "__gm_{scriptName}";
+
+function GM_log(text) {{
+    console.log(text);
+}}
+
+GM_info = (function() {{
+    return {{
+        'script': {scriptInfo},
+        'scriptMetaStr': {scriptMeta},
+        'scriptWillUpdate': false,
+        'version': '0.0.1',
+        'scriptHandler': 'Tampermonkey' //so scripts don't expect exportFunction
+    }};
+}}());
+
+function GM_setValue(key, value) {{
+    if (localStorage !== null &&
+        typeof key === "string" &&
+        (typeof value === "string" ||
+            typeof value === "number" ||
+            typeof value == "boolean")) {{
+        localStorage.setItem(_qute_script_id + key, value);
+    }}
+}}
+
+function GM_getValue(key, default_) {{
+    if (localStorage !== null && typeof key === "string") {{
+        return localStorage.getItem(_qute_script_id + key) || default_;
+    }}
+}}
+
+function GM_deleteValue(key) {{
+    if (localStorage !== null && typeof key === "string") {{
+        localStorage.removeItem(_qute_script_id + key);
+    }}
+}}
+
+function GM_listValues() {{
+    var i;
+    var keys = [];
+    for (i = 0; i < localStorage.length; ++i) {{
+        if (localStorage.key(i).startsWith(_qute_script_id)) {{
+            keys.push(localStorage.key(i));
+        }}
+    }}
+    return keys;
+}}
+
+function GM_openInTab(url) {{
+    window.open(url);
+}}
+
+
+// Almost verbatim copy from Eric
+function GM_xmlhttpRequest(/* object */ details) {{
+    details.method = details.method.toUpperCase() || "GET";
+
+    if(!details.url) {{
+        throw("GM_xmlhttpRequest requires an URL.");
+    }}
+
+    // build XMLHttpRequest object
+    var oXhr = new XMLHttpRequest;
+    // run it
+    if("onreadystatechange" in details)
+        oXhr.onreadystatechange = function() {{
+            details.onreadystatechange(oXhr)
+        }};
+    if("onload" in details)
+        oXhr.onload = function() {{ details.onload(oXhr) }};
+    if("onerror" in details)
+        oXhr.onerror = function() {{ details.onerror(oXhr) }};
+
+    oXhr.open(details.method, details.url, true);
+
+    if("headers" in details)
+        for(var header in details.headers)
+            oXhr.setRequestHeader(header, details.headers[header]);
+
+    if("data" in details)
+        oXhr.send(details.data);
+    else
+        oXhr.send();
+}}
+
+function GM_addStyle(/* String */ styles) {{
+    var head = document.getElementsByTagName("head")[0];
+    if (head === undefined) {{
+        document.onreadystatechange = function() {{
+            if (document.readyState == "interactive") {{
+                var oStyle = document.createElement("style");
+                oStyle.setAttribute("type", "text/css");
+                oStyle.appendChild(document.createTextNode(styles));
+                document.getElementsByTagName("head")[0].appendChild(oStyle);
+            }}
+        }}
+    }}
+    else {{
+        var oStyle = document.createElement("style");
+        oStyle.setAttribute("type", "text/css");
+        oStyle.appendChild(document.createTextNode(styles));
+        head.appendChild(oStyle);
+    }}
+}}
+
+unsafeWindow = window;
+"""
+
+    def __init__(self, properties, code):
+        self._code = code
+        self.includes = []
+        self.excludes = []
+        self.description = None
+        self.name = None
+        self.run_at = None
+        for name, value in properties:
+            if name == 'name':
+                self.name = value
+            elif name == 'description':
+                self.description = value
+            elif name in ['include', 'match']:
+                self.includes.append(value)
+            elif name in ['exclude', 'exclude_match']:
+                self.excludes.append(value)
+            elif name == 'run-at':
+                self.run_at = value
+
+    HEADER_REGEX = r'// ==UserScript==.|\n+// ==/UserScript==\n'
+    PROPS_REGEX = r'// @(?P<prop>[^\s]+)\s+(?P<val>.+)'
+
+    @classmethod
+    def parse(cls, source):
+        """GreaseMonkeyScript factory.
+
+        Takes a userscript source and returns a GreaseMonkeyScript.
+        Parses the greasemonkey metadata block, if present, to fill out
+        attributes.
+        """
+        matches = re.split(cls.HEADER_REGEX, source, maxsplit=1)
+        try:
+            props, _code = matches
+        except ValueError:
+            props = ""
+        script = cls(re.findall(cls.PROPS_REGEX, props), code)
+        script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:]))
+        return script
+
+    def code(self):
+        """Return the processed javascript code of this script.
+
+        Adorns the source code with GM_* methods for greasemonkey
+        compatibility and wraps it in an IFFE to hide it within a
+        lexical scope. Note that this means line numbers in your
+        browser's debugger/inspector will not match up to the line
+        numbers in the source script directly.
+        """
+        gm_bootstrap = self.GM_BOOTSTRAP_TEMPLATE.format(
+            scriptName=self.name,
+            scriptInfo=self._meta_json(),
+            scriptMeta=self.script_meta)
+        return '\n'.join([gm_bootstrap, self._code])
+
+    def _meta_json(self):
+        return json.dumps({
+            'name': self.name,
+            'description': self.description,
+            'matches': self.includes,
+            'includes': self.includes,
+            'excludes': self.excludes,
+            'run-at': self.run_at,
+        })
+
+
+class GreasemonkeyManager:
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._run_start = []
+        self._run_end = []
+
+        scripts_dir = os.path.abspath(_scripts_dir())
+        log.greasemonkey.debug("Reading scripts from: {}".format(scripts_dir))
+        for script_filename in glob.glob(os.path.join(scripts_dir, '*.js')):
+            if not os.path.isfile(script_filename):
+                continue
+            script_path = os.path.join(scripts_dir, script_filename)
+            with open(script_path, encoding='utf-8') as script_file:
+                script = GreasemonkeyScript.parse(script_file.read())
+                if not script.name:
+                    script.name = script_filename
+
+                if script.run_at == 'document-start':
+                    self._run_start.append(script)
+                elif script.run_at == 'document-end':
+                    self._run_end.append(script)
+                else:
+                    log.greasemonkey.warning("Script {} has invalid run-at "
+                                             "defined, ignoring."
+                                             .format(script_path))
+                    continue
+                log.greasemonkey.debug("Loaded script: {}".format(script.name))
+
+    def scripts_for(self, url):
+        """Fetch scripts that are registered to run for url.
+
+        returns a tuple of lists of scripts meant to run at (document-start,
+        document-end)
+        """
+        match = functools.partial(fnmatch.fnmatch, url)
+        tester = (lambda script:
+                  any(map(match, script.includes())) and not
+                  any(map(match, script.excludes())))
+        return (list(filter(tester, self._run_start)),
+                list(filter(tester, self._run_end)))
+
+    def all_scripts(self):
+        """Return all scripts found in the configured script directory."""
+        return self._run_start + self._run_end
diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index 7e1d991b9..9beba6ddc 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -86,6 +86,10 @@ class BrowserPage(QWebPage):
             self.on_save_frame_state_requested)
         self.restoreFrameStateRequested.connect(
             self.on_restore_frame_state_requested)
+        self.mainFrame().javaScriptWindowObjectCleared.connect(
+            functools.partial(self.inject_userjs, load='start'))
+        self.mainFrame().initialLayoutCompleted.connect(
+            functools.partial(self.inject_userjs, load='end'))
 
     def javaScriptPrompt(self, frame, js_msg, default):
         """Override javaScriptPrompt to use qutebrowser prompts."""
@@ -283,6 +287,30 @@ class BrowserPage(QWebPage):
         else:
             self.error_occurred = False
 
+    @pyqtSlot()
+    def inject_userjs(self, load):
+        """Inject user javascripts into the page.
+
+        param: The page load stage to inject the corresponding scripts
+               for. Support values are "start" and "end",
+               corresponding to the allowed values of the `@run-at`
+               directive in the greasemonkey metadata spec.
+        """
+        greasemonkey = objreg.get('greasemonkey')
+        url = self.currentFrame().url()
+        start_scripts, end_scripts = greasemonkey.scripts_for(url.toDisplayString())
+        log.greasemonkey.debug('scripts: {}'.format(start_scripts if start else end_scripts))
+
+        toload = []
+        if load == "start":
+            toload = start_scripts
+        elif load == "end":
+            toload = end_scripts
+
+        for script in toload:
+            log.webview.debug('Running GM script: {}'.format(script.name))
+            self.currentFrame().evaluateJavaScript(script.code())
+
     @pyqtSlot('QWebFrame*', 'QWebPage::Feature')
     def _on_feature_permission_requested(self, frame, feature):
         """Ask the user for approval for geolocation/notifications."""
diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py
index 68cf1d2ba..dc0ff5580 100644
--- a/qutebrowser/utils/log.py
+++ b/qutebrowser/utils/log.py
@@ -95,7 +95,8 @@ LOGGER_NAMES = [
     'commands', 'signals', 'downloads',
     'js', 'qt', 'rfc6266', 'ipc', 'shlexer',
     'save', 'message', 'config', 'sessions',
-    'webelem', 'prompt', 'network', 'sql'
+    'webelem', 'prompt', 'network', 'sql',
+    'greasemonkey'
 ]
 
 
@@ -144,6 +145,7 @@ webelem = logging.getLogger('webelem')
 prompt = logging.getLogger('prompt')
 network = logging.getLogger('network')
 sql = logging.getLogger('sql')
+greasemonkey = logging.getLogger('greasemonkey')
 
 
 ram_handler = None

From ecdde7663ffd44579c5935c5be78d92bc9816885 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 7 Jun 2017 15:40:14 +1200
Subject: [PATCH 02/35] Add greasemonkey-reload command.

Also add a signal to emit when scripts are reloaded. Had to make
GreasemonkeyManager inherit from QObject to get signals to work.
---
 qutebrowser/browser/greasemonkey.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index d1f63a4a4..d01a8f8e5 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -29,6 +29,7 @@ import glob
 from PyQt5.QtCore import pyqtSignal, QObject
 
 from qutebrowser.utils import log, standarddir
+from qutebrowser.commands import cmdutils
 
 # TODO: GM_ bootstrap
 
@@ -215,10 +216,26 @@ unsafeWindow = window;
         })
 
 
-class GreasemonkeyManager:
+class GreasemonkeyManager(QObject):
+
+    """Manager of userscripts and a greasemonkey compatible environment.
+
+    Signals:
+        scripts_reloaded: Emitted when scripts are reloaded from disk.
+            Any any cached or already-injected scripts should be
+            considered obselete.
+    """
+
+    scripts_reloaded = pyqtSignal()
 
     def __init__(self, parent=None):
         super().__init__(parent)
+        self.load_scripts()
+
+    @cmdutils.register(name='greasemonkey-reload',
+                       instance='greasemonkey')
+    def load_scripts(self):
+        """Re-Read greasemonkey scripts from disk."""
         self._run_start = []
         self._run_end = []
 
@@ -243,6 +260,7 @@ class GreasemonkeyManager:
                                              .format(script_path))
                     continue
                 log.greasemonkey.debug("Loaded script: {}".format(script.name))
+        self.scripts_reloaded.emit()
 
     def scripts_for(self, url):
         """Fetch scripts that are registered to run for url.

From 13728387d7d875bbe7bed5687330479e2e4bdc66 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 7 Jun 2017 15:41:53 +1200
Subject: [PATCH 03/35] Greasemonkey: Fix crash on undefined metadata.

---
 qutebrowser/browser/greasemonkey.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index d01a8f8e5..ef8635178 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -256,9 +256,12 @@ class GreasemonkeyManager(QObject):
                     self._run_end.append(script)
                 else:
                     log.greasemonkey.warning("Script {} has invalid run-at "
-                                             "defined, ignoring."
+                                             "defined, defaulting to "
+                                             "document-end"
                                              .format(script_path))
-                    continue
+                    # Default as per
+                    # https://wiki.greasespot.net/Metadata_Block#.40run-at
+                    self._run_end.append(script)
                 log.greasemonkey.debug("Loaded script: {}".format(script.name))
         self.scripts_reloaded.emit()
 

From 25f626a436d1f898391e19fb531984d28bb0547f Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 7 Jun 2017 15:45:12 +1200
Subject: [PATCH 04/35] Greasemonkey: Add run-at document-idle.

Supposed to be after all the assets have finished loading and in page js
has run. Not that we can garuntee that last bit. If a script misbehaves
because a precondition isn't yet met I suggest adding a defer method to
the script that adds a timer until the precondition is met.

Also changed the map/filter calls to use list comprehensions to keep
pylint happy. Even if it does look uglier.
---
 qutebrowser/browser/greasemonkey.py   | 18 ++++++++++++------
 qutebrowser/browser/webkit/webpage.py | 12 +++++++-----
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index ef8635178..058a2d0f4 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -238,6 +238,7 @@ class GreasemonkeyManager(QObject):
         """Re-Read greasemonkey scripts from disk."""
         self._run_start = []
         self._run_end = []
+        self._run_idle = []
 
         scripts_dir = os.path.abspath(_scripts_dir())
         log.greasemonkey.debug("Reading scripts from: {}".format(scripts_dir))
@@ -254,6 +255,8 @@ class GreasemonkeyManager(QObject):
                     self._run_start.append(script)
                 elif script.run_at == 'document-end':
                     self._run_end.append(script)
+                elif script.run_at == 'document-idle':
+                    self._run_idle.append(script)
                 else:
                     log.greasemonkey.warning("Script {} has invalid run-at "
                                              "defined, defaulting to "
@@ -269,15 +272,18 @@ class GreasemonkeyManager(QObject):
         """Fetch scripts that are registered to run for url.
 
         returns a tuple of lists of scripts meant to run at (document-start,
-        document-end)
+        document-end, document-idle)
         """
         match = functools.partial(fnmatch.fnmatch, url)
         tester = (lambda script:
-                  any(map(match, script.includes())) and not
-                  any(map(match, script.excludes())))
-        return (list(filter(tester, self._run_start)),
-                list(filter(tester, self._run_end)))
+                  any([match(pat) for pat in script.includes]) and
+                  not any([match(pat) for pat in script.excludes]))
+        return (
+            [script for script in self._run_start if tester(script)],
+            [script for script in self._run_end if tester(script)],
+            [script for script in self._run_idle if tester(script)]
+        )
 
     def all_scripts(self):
         """Return all scripts found in the configured script directory."""
-        return self._run_start + self._run_end
+        return self._run_start + self._run_end + self._run_idle
diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index 9beba6ddc..095e41fe2 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -90,6 +90,8 @@ class BrowserPage(QWebPage):
             functools.partial(self.inject_userjs, load='start'))
         self.mainFrame().initialLayoutCompleted.connect(
             functools.partial(self.inject_userjs, load='end'))
+        self.mainFrame().loadFinished.connect(
+            functools.partial(self.inject_userjs, load='idle'))
 
     def javaScriptPrompt(self, frame, js_msg, default):
         """Override javaScriptPrompt to use qutebrowser prompts."""
@@ -292,20 +294,20 @@ class BrowserPage(QWebPage):
         """Inject user javascripts into the page.
 
         param: The page load stage to inject the corresponding scripts
-               for. Support values are "start" and "end",
+               for. Support values are "start", "end" and "idle",
                corresponding to the allowed values of the `@run-at`
                directive in the greasemonkey metadata spec.
         """
         greasemonkey = objreg.get('greasemonkey')
         url = self.currentFrame().url()
-        start_scripts, end_scripts = greasemonkey.scripts_for(url.toDisplayString())
-        log.greasemonkey.debug('scripts: {}'.format(start_scripts if start else end_scripts))
-
-        toload = []
+        start_scripts, end_scripts, idle_scripts = \
+            greasemonkey.scripts_for(url.toDisplayString())
         if load == "start":
             toload = start_scripts
         elif load == "end":
             toload = end_scripts
+        elif load == "idle":
+            toload = idle_scripts
 
         for script in toload:
             log.webview.debug('Running GM script: {}'.format(script.name))

From be9f8bd0de2d8189852bb306ebd25c010bc15c76 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 7 Jun 2017 15:49:41 +1200
Subject: [PATCH 05/35] Greasemonkey: Lift greasemonkey init app.py

To prepare for multiple-backend support.
---
 qutebrowser/app.py                | 6 +++++-
 qutebrowser/browser/browsertab.py | 6 +-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/qutebrowser/app.py b/qutebrowser/app.py
index 2ed579f61..c32a208ac 100644
--- a/qutebrowser/app.py
+++ b/qutebrowser/app.py
@@ -64,7 +64,7 @@ from qutebrowser.completion.models import miscmodels
 from qutebrowser.commands import cmdutils, runners, cmdexc
 from qutebrowser.config import config, websettings, configfiles, configinit
 from qutebrowser.browser import (urlmarks, adblock, history, browsertab,
-                                 downloads)
+                                 downloads, greasemonkey)
 from qutebrowser.browser.network import proxy
 from qutebrowser.browser.webkit import cookies, cache
 from qutebrowser.browser.webkit.network import networkmanager
@@ -491,6 +491,10 @@ def _init_modules(args, crash_handler):
     diskcache = cache.DiskCache(standarddir.cache(), parent=qApp)
     objreg.register('cache', diskcache)
 
+    log.init.debug("Initializing Greasemonkey...")
+    gm_manager = greasemonkey.GreasemonkeyManager()
+    objreg.register('greasemonkey', gm_manager)
+
     log.init.debug("Misc initialization...")
     macros.init()
     # Init backend-specific stuff
diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py
index 866943f87..547e276db 100644
--- a/qutebrowser/browser/browsertab.py
+++ b/qutebrowser/browser/browsertab.py
@@ -31,7 +31,7 @@ from qutebrowser.keyinput import modeman
 from qutebrowser.config import config
 from qutebrowser.utils import utils, objreg, usertypes, log, qtutils
 from qutebrowser.misc import miscwidgets, objects
-from qutebrowser.browser import mouse, hints, greasemonkey
+from qutebrowser.browser import mouse, hints
 
 
 tab_id_gen = itertools.count(0)
@@ -64,10 +64,6 @@ def init():
         from qutebrowser.browser.webengine import webenginetab
         webenginetab.init()
 
-    log.init.debug("Initializing Greasemonkey...")
-    gm_manager = greasemonkey.GreasemonkeyManager()
-    objreg.register('greasemonkey', gm_manager)
-
 
 class WebTabError(Exception):
 

From f26377351c0fc8b2e50b583ec7eb8b195d76501c Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 7 Jun 2017 16:06:50 +1200
Subject: [PATCH 06/35] Greasemonkey: Add greasemonkey hooks for webengine.

For qtwebengine 5.8+ only. This is because as of 5.8 some greasemonkey
script support is in upstream. That is, qtwebenginescript(collection)
parses the greasemonkey metadata block and uses @include/match/exclude
to decide what sites to inject a script onto and @run-at to decide when
to inject it, which saves us the trouble. Notes on doing this in <5.8
are below.

Scripts are currently injected into the main "world", that is the same
world as the javascript from the page. This is good because it means
userscripts can modify more stuff on the page but it would be nice if we
could have more isolation without sacrificing functionality. I'm still
looking into why my more feature-full scripts are not having any effect
on the page while running in a separate world.

Userscripts are added to both the default and private profile because I
that if people have scripts installed they want them to run in private mode
too.

We are grabbing the scripts from the greasemonkey module, as opposed to
reading them directly from disk, because the module adds some GM_* functions
that the scripts may expect, and because that is used for webkit anyway.

I have code to support qtwebengine <5.8 but didn't because I am not
happy with the timing of some of the signals that we are provided
regarding page load state, and the actual load state. While the
difference between document-end and document-idle isn't so bad,
injecting document-start scripts when a urlChanged event is emitted
results in the script being injected into the environment for the page
being navigated away from. Anyway, if anyone wants this for earlier
webengines I can oblige them.
---
 qutebrowser/browser/webengine/webenginetab.py | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py
index 89ba958a7..c8d7ef670 100644
--- a/qutebrowser/browser/webengine/webenginetab.py
+++ b/qutebrowser/browser/webengine/webenginetab.py
@@ -69,6 +69,10 @@ def init():
     download_manager.install(webenginesettings.private_profile)
     objreg.register('webengine-download-manager', download_manager)
 
+    greasemonkey = objreg.get('greasemonkey')
+    greasemonkey.scripts_reloaded.connect(inject_userscripts)
+    inject_userscripts()
+
 
 # Mapping worlds from usertypes.JsWorld to QWebEngineScript world IDs.
 _JS_WORLD_MAP = {
@@ -79,6 +83,42 @@ _JS_WORLD_MAP = {
 }
 
 
+def inject_userscripts():
+    """Register user javascript files with the global profiles."""
+    # The greasemonkey metadata block support in qtwebengine only starts at 5.8
+    # Otherwise have to handle injecting the scripts into the page at very
+    # early load, probs same place in view as the enableJS check.
+    if not qtutils.version_check('5.8'):
+        return
+
+    # Since we are inserting scripts into profile.scripts they won't
+    # just get replaced by new gm scripts like if we were injecting them
+    # ourselves so we need to remove all gm scripts, while not removing
+    # any other stuff that might have been added. Like the one for
+    # stylsheets.
+    # Could either use a different world for gm scripts, check for gm metadata
+    # values (would mean no non-gm userscripts), or check the code for
+    # _qute_script_id
+    for profile in [webenginesettings.default_profile,
+                    webenginesettings.private_profile]:
+        scripts = profile.scripts()
+        for script in scripts.toList():
+            if script.worldId() == QWebEngineScript.MainWorld:
+                scripts.remove(script)
+
+    # Should we be adding to private profile too?
+    for profile in [webenginesettings.default_profile,
+                    webenginesettings.private_profile]:
+        scripts = profile.scripts()
+        greasemonkey = objreg.get('greasemonkey')
+        for script in greasemonkey.all_scripts():
+            new_script = QWebEngineScript()
+            new_script.setWorldId(QWebEngineScript.MainWorld)
+            new_script.setSourceCode(script.code())
+            log.greasemonkey.debug('adding script: %s', new_script.name)
+            scripts.insert(new_script)
+
+
 class WebEngineAction(browsertab.AbstractAction):
 
     """QtWebEngine implementations related to web actions."""

From 325c595b896ded73d324b8a2a9c178b5ef1fd604 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Thu, 27 Jul 2017 19:25:29 +1200
Subject: [PATCH 07/35] Greasemonkey: Don't strip gm metadata from scripts when
 loading.

Since we just pass them to webenginescriptcollection on that backend and
that wants to parse it itself to figure out injection point etc.
---
 qutebrowser/browser/greasemonkey.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index 058a2d0f4..a616891d1 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -31,8 +31,6 @@ from PyQt5.QtCore import pyqtSignal, QObject
 from qutebrowser.utils import log, standarddir
 from qutebrowser.commands import cmdutils
 
-# TODO: GM_ bootstrap
-
 
 def _scripts_dir():
     """Get the directory of the scripts."""
@@ -186,7 +184,7 @@ unsafeWindow = window;
             props, _code = matches
         except ValueError:
             props = ""
-        script = cls(re.findall(cls.PROPS_REGEX, props), code)
+        script = cls(re.findall(cls.PROPS_REGEX, props), source)
         script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:]))
         return script
 

From 799730f6864ad8f41365542d5c393ecbd52f7155 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Thu, 27 Jul 2017 21:21:21 +1200
Subject: [PATCH 08/35] Remove GM_ and userscript variables from global scope.

---
 qutebrowser/browser/greasemonkey.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index a616891d1..41d32bcf1 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -201,7 +201,8 @@ unsafeWindow = window;
             scriptName=self.name,
             scriptInfo=self._meta_json(),
             scriptMeta=self.script_meta)
-        return '\n'.join([gm_bootstrap, self._code])
+        return '\n'.join(
+            ["(function(){", gm_bootstrap, self._code, "})();"])
 
     def _meta_json(self):
         return json.dumps({

From 41035cb5cab72ab3c4acfb06330eab6e7b8f25b4 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Sun, 5 Nov 2017 16:36:09 +1300
Subject: [PATCH 09/35] Greasemonkey: restrict page schemes that scripts can
 run on

Scripts shouldn't run on qute://settings or source:// etc.

Whitelist from:
https://wiki.greasespot.net/Include_and_exclude_rules
---
 qutebrowser/browser/greasemonkey.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index 41d32bcf1..4a8d9bdeb 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -226,6 +226,10 @@ class GreasemonkeyManager(QObject):
     """
 
     scripts_reloaded = pyqtSignal()
+    # https://wiki.greasespot.net/Include_and_exclude_rules#Greaseable_schemes
+    # Limit the schemes scripts can run on due to unreasonable levels of
+    # exploitability
+    greaseable_schemes = ['http', 'https', 'ftp', 'file']
 
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -273,6 +277,8 @@ class GreasemonkeyManager(QObject):
         returns a tuple of lists of scripts meant to run at (document-start,
         document-end, document-idle)
         """
+        if url.split(':', 1)[0] not in self.greaseable_schemes:
+            return [], [], []
         match = functools.partial(fnmatch.fnmatch, url)
         tester = (lambda script:
                   any([match(pat) for pat in script.includes]) and

From edf737ff7d604a3d14a26eb23b0b0a94f9328230 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 4 Oct 2017 20:42:10 +1300
Subject: [PATCH 10/35] Greasemonkey: move scripts for a domain into data
 class.

Also makes scripts that don't include a greasemonkey metadata block
match any url. QWebEngine already has that behaviour.
---
 qutebrowser/browser/greasemonkey.py   | 18 ++++++++++++++++--
 qutebrowser/browser/webkit/webpage.py | 10 +++++-----
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index 4a8d9bdeb..d06bc9cac 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -26,6 +26,7 @@ import fnmatch
 import functools
 import glob
 
+import attr
 from PyQt5.QtCore import pyqtSignal, QObject
 
 from qutebrowser.utils import log, standarddir
@@ -186,6 +187,8 @@ unsafeWindow = window;
             props = ""
         script = cls(re.findall(cls.PROPS_REGEX, props), source)
         script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:]))
+        if not props:
+            script.includes = ['*']
         return script
 
     def code(self):
@@ -215,6 +218,16 @@ unsafeWindow = window;
         })
 
 
+@attr.s
+class MatchingScripts(object):
+    """All userscripts registered to run on a particular url."""
+
+    url = attr.ib()
+    start = attr.ib(default=attr.Factory(list))
+    end = attr.ib(default=attr.Factory(list))
+    idle = attr.ib(default=attr.Factory(list))
+
+
 class GreasemonkeyManager(QObject):
 
     """Manager of userscripts and a greasemonkey compatible environment.
@@ -278,12 +291,13 @@ class GreasemonkeyManager(QObject):
         document-end, document-idle)
         """
         if url.split(':', 1)[0] not in self.greaseable_schemes:
-            return [], [], []
+            return MatchingScripts(url, [], [], [])
         match = functools.partial(fnmatch.fnmatch, url)
         tester = (lambda script:
                   any([match(pat) for pat in script.includes]) and
                   not any([match(pat) for pat in script.excludes]))
-        return (
+        return MatchingScripts(
+            url,
             [script for script in self._run_start if tester(script)],
             [script for script in self._run_end if tester(script)],
             [script for script in self._run_idle if tester(script)]
diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index 095e41fe2..15f662e61 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -300,14 +300,14 @@ class BrowserPage(QWebPage):
         """
         greasemonkey = objreg.get('greasemonkey')
         url = self.currentFrame().url()
-        start_scripts, end_scripts, idle_scripts = \
-            greasemonkey.scripts_for(url.toDisplayString())
+        scripts = greasemonkey.scripts_for(url.toDisplayString())
+
         if load == "start":
-            toload = start_scripts
+            toload = scripts.start
         elif load == "end":
-            toload = end_scripts
+            toload = scripts.end
         elif load == "idle":
-            toload = idle_scripts
+            toload = scripts.idle
 
         for script in toload:
             log.webview.debug('Running GM script: {}'.format(script.name))

From c1b912f5670756b6a96ba378e60e83e9df64ba98 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 4 Oct 2017 21:24:16 +1300
Subject: [PATCH 11/35] Greasemonkey: move inject_userscripts into
 webenginesettings

---
 .../browser/webengine/webenginesettings.py    | 33 +++++++++++++++
 qutebrowser/browser/webengine/webenginetab.py | 40 +------------------
 2 files changed, 35 insertions(+), 38 deletions(-)

diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py
index 4bf525c46..5ce4e1a05 100644
--- a/qutebrowser/browser/webengine/webenginesettings.py
+++ b/qutebrowser/browser/webengine/webenginesettings.py
@@ -244,6 +244,39 @@ def _init_profiles():
         private_profile.setSpellCheckEnabled(True)
 
 
+def inject_userscripts():
+    """Register user javascript files with the global profiles."""
+    # The greasemonkey metadata block support in qtwebengine only starts at 5.8
+    # Otherwise have to handle injecting the scripts into the page at very
+    # early load, probs same place in view as the enableJS check.
+    if not qtutils.version_check('5.8'):
+        return
+
+    # Since we are inserting scripts into profile.scripts they won't
+    # just get replaced by new gm scripts like if we were injecting them
+    # ourselves so we need to remove all gm scripts, while not removing
+    # any other stuff that might have been added. Like the one for
+    # stylsheets.
+    # Could either use a different world for gm scripts, check for gm metadata
+    # values (would mean no non-gm userscripts), or check the code for
+    # _qute_script_id
+    for profile in [default_profile, private_profile]:
+        scripts = profile.scripts()
+        for script in scripts.toList():
+            if script.worldId() == QWebEngineScript.MainWorld:
+                scripts.remove(script)
+
+    for profile in [default_profile, private_profile]:
+        scripts = profile.scripts()
+        greasemonkey = objreg.get('greasemonkey')
+        for script in greasemonkey.all_scripts():
+            new_script = QWebEngineScript()
+            new_script.setWorldId(QWebEngineScript.MainWorld)
+            new_script.setSourceCode(script.code())
+            log.greasemonkey.debug('adding script: %s', new_script.name())
+            scripts.insert(new_script)
+
+
 def init(args):
     """Initialize the global QWebSettings."""
     if args.enable_webengine_inspector:
diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py
index c8d7ef670..c97aaacbe 100644
--- a/qutebrowser/browser/webengine/webenginetab.py
+++ b/qutebrowser/browser/webengine/webenginetab.py
@@ -70,8 +70,8 @@ def init():
     objreg.register('webengine-download-manager', download_manager)
 
     greasemonkey = objreg.get('greasemonkey')
-    greasemonkey.scripts_reloaded.connect(inject_userscripts)
-    inject_userscripts()
+    greasemonkey.scripts_reloaded.connect(webenginesettings.inject_userscripts)
+    webenginesettings.inject_userscripts()
 
 
 # Mapping worlds from usertypes.JsWorld to QWebEngineScript world IDs.
@@ -83,42 +83,6 @@ _JS_WORLD_MAP = {
 }
 
 
-def inject_userscripts():
-    """Register user javascript files with the global profiles."""
-    # The greasemonkey metadata block support in qtwebengine only starts at 5.8
-    # Otherwise have to handle injecting the scripts into the page at very
-    # early load, probs same place in view as the enableJS check.
-    if not qtutils.version_check('5.8'):
-        return
-
-    # Since we are inserting scripts into profile.scripts they won't
-    # just get replaced by new gm scripts like if we were injecting them
-    # ourselves so we need to remove all gm scripts, while not removing
-    # any other stuff that might have been added. Like the one for
-    # stylsheets.
-    # Could either use a different world for gm scripts, check for gm metadata
-    # values (would mean no non-gm userscripts), or check the code for
-    # _qute_script_id
-    for profile in [webenginesettings.default_profile,
-                    webenginesettings.private_profile]:
-        scripts = profile.scripts()
-        for script in scripts.toList():
-            if script.worldId() == QWebEngineScript.MainWorld:
-                scripts.remove(script)
-
-    # Should we be adding to private profile too?
-    for profile in [webenginesettings.default_profile,
-                    webenginesettings.private_profile]:
-        scripts = profile.scripts()
-        greasemonkey = objreg.get('greasemonkey')
-        for script in greasemonkey.all_scripts():
-            new_script = QWebEngineScript()
-            new_script.setWorldId(QWebEngineScript.MainWorld)
-            new_script.setSourceCode(script.code())
-            log.greasemonkey.debug('adding script: %s', new_script.name)
-            scripts.insert(new_script)
-
-
 class WebEngineAction(browsertab.AbstractAction):
 
     """QtWebEngine implementations related to web actions."""

From fd5d44182ba53e6e3c5fb8c71ffe15d3c44aeeaa Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 4 Oct 2017 22:39:32 +1300
Subject: [PATCH 12/35] Greasemonkey: move GM_* template into seperate file.

Also ported it to jinja rather than str.format().

Also ran the js through jslint and fixed up a few very minor things.
---
 qutebrowser/browser/greasemonkey.py           | 123 +-----------------
 qutebrowser/javascript/.eslintignore          |   2 +
 .../javascript/greasemonkey_wrapper.js        | 118 +++++++++++++++++
 qutebrowser/utils/jinja.py                    |   1 +
 4 files changed, 128 insertions(+), 116 deletions(-)
 create mode 100644 qutebrowser/javascript/greasemonkey_wrapper.js

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index d06bc9cac..e1f0b57db 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -29,7 +29,7 @@ import glob
 import attr
 from PyQt5.QtCore import pyqtSignal, QObject
 
-from qutebrowser.utils import log, standarddir
+from qutebrowser.utils import log, standarddir, jinja
 from qutebrowser.commands import cmdutils
 
 
@@ -41,115 +41,6 @@ def _scripts_dir():
 class GreasemonkeyScript:
     """Container class for userscripts, parses metadata blocks."""
 
-    GM_BOOTSTRAP_TEMPLATE = r"""var _qute_script_id = "__gm_{scriptName}";
-
-function GM_log(text) {{
-    console.log(text);
-}}
-
-GM_info = (function() {{
-    return {{
-        'script': {scriptInfo},
-        'scriptMetaStr': {scriptMeta},
-        'scriptWillUpdate': false,
-        'version': '0.0.1',
-        'scriptHandler': 'Tampermonkey' //so scripts don't expect exportFunction
-    }};
-}}());
-
-function GM_setValue(key, value) {{
-    if (localStorage !== null &&
-        typeof key === "string" &&
-        (typeof value === "string" ||
-            typeof value === "number" ||
-            typeof value == "boolean")) {{
-        localStorage.setItem(_qute_script_id + key, value);
-    }}
-}}
-
-function GM_getValue(key, default_) {{
-    if (localStorage !== null && typeof key === "string") {{
-        return localStorage.getItem(_qute_script_id + key) || default_;
-    }}
-}}
-
-function GM_deleteValue(key) {{
-    if (localStorage !== null && typeof key === "string") {{
-        localStorage.removeItem(_qute_script_id + key);
-    }}
-}}
-
-function GM_listValues() {{
-    var i;
-    var keys = [];
-    for (i = 0; i < localStorage.length; ++i) {{
-        if (localStorage.key(i).startsWith(_qute_script_id)) {{
-            keys.push(localStorage.key(i));
-        }}
-    }}
-    return keys;
-}}
-
-function GM_openInTab(url) {{
-    window.open(url);
-}}
-
-
-// Almost verbatim copy from Eric
-function GM_xmlhttpRequest(/* object */ details) {{
-    details.method = details.method.toUpperCase() || "GET";
-
-    if(!details.url) {{
-        throw("GM_xmlhttpRequest requires an URL.");
-    }}
-
-    // build XMLHttpRequest object
-    var oXhr = new XMLHttpRequest;
-    // run it
-    if("onreadystatechange" in details)
-        oXhr.onreadystatechange = function() {{
-            details.onreadystatechange(oXhr)
-        }};
-    if("onload" in details)
-        oXhr.onload = function() {{ details.onload(oXhr) }};
-    if("onerror" in details)
-        oXhr.onerror = function() {{ details.onerror(oXhr) }};
-
-    oXhr.open(details.method, details.url, true);
-
-    if("headers" in details)
-        for(var header in details.headers)
-            oXhr.setRequestHeader(header, details.headers[header]);
-
-    if("data" in details)
-        oXhr.send(details.data);
-    else
-        oXhr.send();
-}}
-
-function GM_addStyle(/* String */ styles) {{
-    var head = document.getElementsByTagName("head")[0];
-    if (head === undefined) {{
-        document.onreadystatechange = function() {{
-            if (document.readyState == "interactive") {{
-                var oStyle = document.createElement("style");
-                oStyle.setAttribute("type", "text/css");
-                oStyle.appendChild(document.createTextNode(styles));
-                document.getElementsByTagName("head")[0].appendChild(oStyle);
-            }}
-        }}
-    }}
-    else {{
-        var oStyle = document.createElement("style");
-        oStyle.setAttribute("type", "text/css");
-        oStyle.appendChild(document.createTextNode(styles));
-        head.appendChild(oStyle);
-    }}
-}}
-
-unsafeWindow = window;
-"""
-
     def __init__(self, properties, code):
         self._code = code
         self.includes = []
@@ -200,12 +91,12 @@ unsafeWindow = window;
         browser's debugger/inspector will not match up to the line
         numbers in the source script directly.
         """
-        gm_bootstrap = self.GM_BOOTSTRAP_TEMPLATE.format(
-            scriptName=self.name,
-            scriptInfo=self._meta_json(),
-            scriptMeta=self.script_meta)
-        return '\n'.join(
-            ["(function(){", gm_bootstrap, self._code, "})();"])
+        return jinja.js_environment.get_template(
+            'greasemonkey_wrapper.js').render(
+                scriptName=self.name,
+                scriptInfo=self._meta_json(),
+                scriptMeta=self.script_meta,
+                scriptSource=self._code)
 
     def _meta_json(self):
         return json.dumps({
diff --git a/qutebrowser/javascript/.eslintignore b/qutebrowser/javascript/.eslintignore
index ca4d3c667..036a72cfe 100644
--- a/qutebrowser/javascript/.eslintignore
+++ b/qutebrowser/javascript/.eslintignore
@@ -1,2 +1,4 @@
 # Upstream Mozilla's code
 pac_utils.js
+# Actually a jinja template so eslint chokes on the {{}} syntax.
+greasemonkey_wrapper.js
diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js
new file mode 100644
index 000000000..b49dc2c02
--- /dev/null
+++ b/qutebrowser/javascript/greasemonkey_wrapper.js
@@ -0,0 +1,118 @@
+(function () {
+    var _qute_script_id = "__gm_{{ scriptName }}";
+
+    function GM_log(text) {
+        console.log(text);
+    }
+
+    var GM_info = (function () {
+        return {
+            'script': {{ scriptInfo }},
+            'scriptMetaStr': {{ scriptMeta }},
+            'scriptWillUpdate': false,
+            'version': '0.0.1',
+            'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction
+        };
+    }());
+
+    function GM_setValue(key, value) {
+        if (localStorage !== null &&
+                typeof key === "string" &&
+                (typeof value === "string" ||
+                 typeof value === "number" ||
+                 typeof value === "boolean")) {
+            localStorage.setItem(_qute_script_id + key, value);
+        }
+    }
+
+    function GM_getValue(key, default_) {
+        if (localStorage !== null && typeof key === "string") {
+            return localStorage.getItem(_qute_script_id + key) || default_;
+        }
+    }
+
+    function GM_deleteValue(key) {
+        if (localStorage !== null && typeof key === "string") {
+            localStorage.removeItem(_qute_script_id + key);
+        }
+    }
+
+    function GM_listValues() {
+        var i, keys = [];
+        for (i = 0; i < localStorage.length; i = i + 1) {
+            if (localStorage.key(i).startsWith(_qute_script_id)) {
+                keys.push(localStorage.key(i));
+            }
+        }
+        return keys;
+    }
+
+    function GM_openInTab(url) {
+        window.open(url);
+    }
+
+
+    // Almost verbatim copy from Eric
+    function GM_xmlhttpRequest(/* object */ details) {
+        details.method = details.method.toUpperCase() || "GET";
+
+        if (!details.url) {
+            throw ("GM_xmlhttpRequest requires an URL.");
+        }
+
+        // build XMLHttpRequest object
+        var oXhr = new XMLHttpRequest();
+        // run it
+        if ("onreadystatechange" in details) {
+            oXhr.onreadystatechange = function () {
+                details.onreadystatechange(oXhr);
+            };
+        }
+        if ("onload" in details) {
+            oXhr.onload = function () { details.onload(oXhr) };
+        }
+        if ("onerror" in details) {
+            oXhr.onerror = function () { details.onerror(oXhr) };
+        }
+
+        oXhr.open(details.method, details.url, true);
+
+        if ("headers" in details) {
+            for (var header in details.headers) {
+                oXhr.setRequestHeader(header, details.headers[header]);
+            }
+        }
+
+        if ("data" in details) {
+            oXhr.send(details.data);
+        } else {
+            oXhr.send();
+        }
+    }
+
+    function GM_addStyle(/* String */ styles) {
+        var head = document.getElementsByTagName("head")[0];
+        if (head === undefined) {
+            document.onreadystatechange = function () {
+                if (document.readyState == "interactive") {
+                    var oStyle = document.createElement("style");
+                    oStyle.setAttribute("type", "text/css");
+                    oStyle.appendChild(document.createTextNode(styles));
+                    document.getElementsByTagName("head")[0].appendChild(oStyle);
+                }
+            }
+        }
+        else {
+            var oStyle = document.createElement("style");
+            oStyle.setAttribute("type", "text/css");
+            oStyle.appendChild(document.createTextNode(styles));
+            head.appendChild(oStyle);
+        }
+    }
+
+    unsafeWindow = window;
+
+    //====== The actual user script source ======//
+{{ scriptSource }}
+    //====== End User Script ======//
+})();
diff --git a/qutebrowser/utils/jinja.py b/qutebrowser/utils/jinja.py
index e7b536b60..b6f53645b 100644
--- a/qutebrowser/utils/jinja.py
+++ b/qutebrowser/utils/jinja.py
@@ -136,3 +136,4 @@ def render(template, **kwargs):
 
 
 environment = Environment()
+js_environment = jinja2.Environment(loader=Loader('javascript'))

From a7f41b4564dd39247893662ebeb23170a290c85f Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 4 Oct 2017 23:25:22 +1300
Subject: [PATCH 13/35] Greasemonkey: ensure only GM scripts are cleaned up on
 reload.

WebEngine only. Previously we were just removing every script from the
main world. But some other scripts might got here in the future so new
we are overriding the name field to add a GM- prefix so hopefully we
only remove greasemonkey scripts before adding new ones.
---
 qutebrowser/browser/webengine/webenginesettings.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py
index 5ce4e1a05..18ba98fd6 100644
--- a/qutebrowser/browser/webengine/webenginesettings.py
+++ b/qutebrowser/browser/webengine/webenginesettings.py
@@ -263,7 +263,9 @@ def inject_userscripts():
     for profile in [default_profile, private_profile]:
         scripts = profile.scripts()
         for script in scripts.toList():
-            if script.worldId() == QWebEngineScript.MainWorld:
+            if script.name().startswith("GM-"):
+                log.greasemonkey.debug('removing script: {}'
+                                       .format(script.name()))
                 scripts.remove(script)
 
     for profile in [default_profile, private_profile]:
@@ -273,6 +275,7 @@ def inject_userscripts():
             new_script = QWebEngineScript()
             new_script.setWorldId(QWebEngineScript.MainWorld)
             new_script.setSourceCode(script.code())
+            new_script.setName("GM-{}".format(script.name))
             log.greasemonkey.debug('adding script: %s', new_script.name())
             scripts.insert(new_script)
 

From d93c583c0d8f1859be2e306f21477ee2444b2d83 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Sat, 7 Oct 2017 17:18:48 +1300
Subject: [PATCH 14/35] Greasemonkey: Escape jinja variables for JS strings.

---
 qutebrowser/javascript/greasemonkey_wrapper.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js
index b49dc2c02..a5079b89a 100644
--- a/qutebrowser/javascript/greasemonkey_wrapper.js
+++ b/qutebrowser/javascript/greasemonkey_wrapper.js
@@ -1,5 +1,5 @@
 (function () {
-    var _qute_script_id = "__gm_{{ scriptName }}";
+    var _qute_script_id = "__gm_"+{{ scriptName | tojson }};
 
     function GM_log(text) {
         console.log(text);
@@ -7,8 +7,8 @@
 
     var GM_info = (function () {
         return {
-            'script': {{ scriptInfo }},
-            'scriptMetaStr': {{ scriptMeta }},
+            'script': {{ scriptInfo | tojson }},
+            'scriptMetaStr': {{ scriptMeta | tojson }},
             'scriptWillUpdate': false,
             'version': '0.0.1',
             'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction

From 5e49e7eef21ef1abaf327b1c060145f67ba6f868 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Sat, 7 Oct 2017 17:22:57 +1300
Subject: [PATCH 15/35] Greasemonkey: Throw Errors if GM_ function args wrong
 type.

These argument type restrictions are mentioned on the greasespot pages for
these value storage functions. We could call JSON.dumps() instead but better
to push that onto the caller so we don't have to try handle deserialization.

Also removes the check for localstorage because everyone has supported that
for years.
---
 .../javascript/greasemonkey_wrapper.js        | 24 +++++++++++--------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js
index a5079b89a..605f82d5a 100644
--- a/qutebrowser/javascript/greasemonkey_wrapper.js
+++ b/qutebrowser/javascript/greasemonkey_wrapper.js
@@ -16,25 +16,29 @@
     }());
 
     function GM_setValue(key, value) {
-        if (localStorage !== null &&
-                typeof key === "string" &&
-                (typeof value === "string" ||
-                 typeof value === "number" ||
-                 typeof value === "boolean")) {
-            localStorage.setItem(_qute_script_id + key, value);
+        if (typeof key !== "string") {
+          throw new Error("GM_setValue requires the first parameter to be of type string, not '"+typeof key+"'");
         }
+        if (typeof value !== "string" ||
+            typeof value !== "number" ||
+            typeof value !== "boolean") {
+          throw new Error("GM_setValue requires the second parameter to be of type string, number or boolean, not '"+typeof value+"'");
+        }
+        localStorage.setItem(_qute_script_id + key, value);
     }
 
     function GM_getValue(key, default_) {
-        if (localStorage !== null && typeof key === "string") {
-            return localStorage.getItem(_qute_script_id + key) || default_;
+        if (typeof key !== "string") {
+          throw new Error("GM_getValue requires the first parameter to be of type string, not '"+typeof key+"'");
         }
+        return localStorage.getItem(_qute_script_id + key) || default_;
     }
 
     function GM_deleteValue(key) {
-        if (localStorage !== null && typeof key === "string") {
-            localStorage.removeItem(_qute_script_id + key);
+        if (typeof key !== "string") {
+          throw new Error("GM_deleteValue requires the first parameter to be of type string, not '"+typeof key+"'");
         }
+        localStorage.removeItem(_qute_script_id + key);
     }
 
     function GM_listValues() {

From d318178567f8e4236ded6ff350628127d4be7496 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Sat, 7 Oct 2017 17:32:21 +1300
Subject: [PATCH 16/35] Greasemonkey: Fix metadata block regex.

This regex was broken since the original PR and subsequent code seemed to be
working around it. Before re.split was returning [everything up to
/UserScript, everything else], now it returns [before UserScript, metadata,
after /UserScript], which is good.

Also I added the check for the UserScript line starting at column 0 as per
spec.
---
 qutebrowser/browser/greasemonkey.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index e1f0b57db..a0eb109fd 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -60,7 +60,7 @@ class GreasemonkeyScript:
             elif name == 'run-at':
                 self.run_at = value
 
-    HEADER_REGEX = r'// ==UserScript==.|\n+// ==/UserScript==\n'
+    HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n'
     PROPS_REGEX = r'// @(?P<prop>[^\s]+)\s+(?P<val>.+)'
 
     @classmethod
@@ -71,13 +71,13 @@ class GreasemonkeyScript:
         Parses the greasemonkey metadata block, if present, to fill out
         attributes.
         """
-        matches = re.split(cls.HEADER_REGEX, source, maxsplit=1)
+        matches = re.split(cls.HEADER_REGEX, source, maxsplit=2)
         try:
-            props, _code = matches
+            _, props, _code = matches
         except ValueError:
             props = ""
         script = cls(re.findall(cls.PROPS_REGEX, props), source)
-        script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:]))
+        script.script_meta = props
         if not props:
             script.includes = ['*']
         return script

From 209e43e0baa0eda4c3258abc2dad75618020db99 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Sat, 7 Oct 2017 17:33:00 +1300
Subject: [PATCH 17/35] Greasemonkey: Match against percent encoded urls only.

This change requires urls specified in @include, @exclude and @matches
directives in metadata blocks to be in the same form that
QUrl.toEncoded() returns. That is a punycoded domain and percent encoded
path and query. This seems to be what Tampermonkey on chrome expects to.

Also changes the scripts_for() function to take a QUrl arg so the caller
doesn't need to worry about encodings.
---
 qutebrowser/browser/greasemonkey.py   | 5 +++--
 qutebrowser/browser/webkit/webpage.py | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index a0eb109fd..3243ed8d8 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -181,9 +181,10 @@ class GreasemonkeyManager(QObject):
         returns a tuple of lists of scripts meant to run at (document-start,
         document-end, document-idle)
         """
-        if url.split(':', 1)[0] not in self.greaseable_schemes:
+        if url.scheme() not in self.greaseable_schemes:
             return MatchingScripts(url, [], [], [])
-        match = functools.partial(fnmatch.fnmatch, url)
+        match = functools.partial(fnmatch.fnmatch,
+                                  str(url.toEncoded(), 'utf-8'))
         tester = (lambda script:
                   any([match(pat) for pat in script.includes]) and
                   not any([match(pat) for pat in script.excludes]))
diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index 15f662e61..e48519189 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -300,7 +300,7 @@ class BrowserPage(QWebPage):
         """
         greasemonkey = objreg.get('greasemonkey')
         url = self.currentFrame().url()
-        scripts = greasemonkey.scripts_for(url.toDisplayString())
+        scripts = greasemonkey.scripts_for(url)
 
         if load == "start":
             toload = scripts.start

From efde31aa5700f41b75ee7f119d6c3cacea9cdf90 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Sun, 8 Oct 2017 12:35:01 +1300
Subject: [PATCH 18/35] Greasemonkey: Support QTWebEngine versions < 5.8

QTWebEngine 5.8 added support for parsing greasemonkey metadata blocks
and scripts added to the QWebEngineScriptCollection of a page or its
profile and then deciding what urls to run those scripts on and at what
point in the load process to run them. For earlier versions we must do
that work ourselves. But with the additional handicap of the less rich
qtwebengine api.

We have acceptNavigationRequest, loadStarted, loadProgress,
loadFinished, urlChanged to choose from regarding points at which to
register scripts for the current page.

Adding scripts on acceptNavigation loadStarted and loadFinished causes
scripts to run too early or too late (eg on the pages being navigated
from/to) and not run on the desired page at the time they are inserted.
We could maybe do some more sophisticated stuff with loadProgress but it
didn't have any better behaviour in the brief testing I gave it.
Registering scripts on the urlChanged event seems to work fine. Even if
it seems like there could be problems with the signal firing too often,
due to not necessarily being tied to the page load progress, that
doesn't seem to have an effect in practice. The event is fired when, for
example, the url fragment changes and even if we add a new script to the
collection (or remove an existing one) it doesn't have an effect on what
is running on the page.

I suspect all of those timing issues is due to the signals being
forwarded fairly directly from the underlying chomium/blink code but the
webengine script stuff only being pushed back to the implementation on
certain events.

Anyway, using urlChanged seems to work fine due to some quirk(s) of the
implementation. That might change with later development but this
codepath is only ever going to be used for version 5.7.

There are other potential optimizations like not removing and then
re-adding scripts for the current page. But they probably wouldn't do
anything anyway, or at least anything that you would expect.
---
 qutebrowser/browser/webengine/webview.py | 45 ++++++++++++++++++++++--
 1 file changed, 43 insertions(+), 2 deletions(-)

diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py
index 56bd1eb5a..35d6fca40 100644
--- a/qutebrowser/browser/webengine/webview.py
+++ b/qutebrowser/browser/webengine/webview.py
@@ -23,12 +23,14 @@ import functools
 
 from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, PYQT_VERSION
 from PyQt5.QtGui import QPalette
-from PyQt5.QtWebEngineWidgets import QWebEngineView, QWebEnginePage
+from PyQt5.QtWebEngineWidgets import (QWebEngineView, QWebEnginePage,
+                                      QWebEngineScript)
 
 from qutebrowser.browser import shared
 from qutebrowser.browser.webengine import certificateerror, webenginesettings
 from qutebrowser.config import config
-from qutebrowser.utils import log, debug, usertypes, jinja, urlutils, message
+from qutebrowser.utils import (log, debug, usertypes, jinja, urlutils, message,
+                               objreg, qtutils)
 
 
 class WebEngineView(QWebEngineView):
@@ -135,6 +137,7 @@ class WebEnginePage(QWebEnginePage):
         self._theme_color = theme_color
         self._set_bg_color()
         config.instance.changed.connect(self._set_bg_color)
+        self.urlChanged.connect(self._inject_userjs)
 
     @config.change_filter('colors.webpage.bg')
     def _set_bg_color(self):
@@ -300,3 +303,41 @@ class WebEnginePage(QWebEnginePage):
             message.error(msg)
             return False
         return True
+
+    @pyqtSlot('QUrl')
+    def _inject_userjs(self, url):
+        """Inject userscripts registered for `url` into the current page."""
+        if qtutils.version_check('5.8'):
+            # Handled in webenginetab with the builtin greasemonkey
+            # support.
+            return
+
+        # Using QWebEnginePage.scripts() to hold the user scripts means
+        # we don't have to worry ourselves about where to inject the
+        # page but also means scripts hang around for the tab lifecycle.
+        # So clear them here.
+        scripts = self.scripts()
+        for script in scripts.toList():
+            if script.name().startswith("GM-"):
+                really_removed = scripts.remove(script)
+                log.greasemonkey.debug("Removing ({}) script: {}"
+                                       .format(really_removed, script.name()))
+
+        def _add_script(script, injection_point):
+            new_script = QWebEngineScript()
+            new_script.setInjectionPoint(injection_point)
+            new_script.setWorldId(QWebEngineScript.MainWorld)
+            new_script.setSourceCode(script.code())
+            new_script.setName("GM-{}".format(script.name))
+            log.greasemonkey.debug("Adding script: {}"
+                                   .format(new_script.name()))
+            scripts.insert(new_script)
+
+        greasemonkey = objreg.get('greasemonkey')
+        matching_scripts = greasemonkey.scripts_for(url)
+        for script in matching_scripts.start:
+            _add_script(script, QWebEngineScript.DocumentCreation)
+        for script in matching_scripts.end:
+            _add_script(script, QWebEngineScript.DocumentReady)
+        for script in matching_scripts.idle:
+            _add_script(script, QWebEngineScript.Deferred)

From fb019b2dab342a98ae8e364e893650fecd42fd62 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Tue, 10 Oct 2017 20:45:10 +1300
Subject: [PATCH 19/35] Address second round line comments.

Add qute version to GM_info object in GM wrapper.
Support using the greasemonkey @namespace metadata for its intended
purpose of avoiding name collisions.
Get a nice utf8 encoded string from a QUrl more better.
---
 qutebrowser/browser/greasemonkey.py           |  9 +++++---
 .../javascript/greasemonkey_wrapper.js        | 22 +++++++++++--------
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index 3243ed8d8..b58a558a5 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -27,7 +27,7 @@ import functools
 import glob
 
 import attr
-from PyQt5.QtCore import pyqtSignal, QObject
+from PyQt5.QtCore import pyqtSignal, QObject, QUrl
 
 from qutebrowser.utils import log, standarddir, jinja
 from qutebrowser.commands import cmdutils
@@ -47,10 +47,13 @@ class GreasemonkeyScript:
         self.excludes = []
         self.description = None
         self.name = None
+        self.namespace = None
         self.run_at = None
         for name, value in properties:
             if name == 'name':
                 self.name = value
+            elif name == 'namespace':
+                self.namespace = value
             elif name == 'description':
                 self.description = value
             elif name in ['include', 'match']:
@@ -93,7 +96,7 @@ class GreasemonkeyScript:
         """
         return jinja.js_environment.get_template(
             'greasemonkey_wrapper.js').render(
-                scriptName=self.name,
+                scriptName="/".join([self.namespace or '', self.name]),
                 scriptInfo=self._meta_json(),
                 scriptMeta=self.script_meta,
                 scriptSource=self._code)
@@ -184,7 +187,7 @@ class GreasemonkeyManager(QObject):
         if url.scheme() not in self.greaseable_schemes:
             return MatchingScripts(url, [], [], [])
         match = functools.partial(fnmatch.fnmatch,
-                                  str(url.toEncoded(), 'utf-8'))
+                                  url.toString(QUrl.FullyEncoded))
         tester = (lambda script:
                   any([match(pat) for pat in script.includes]) and
                   not any([match(pat) for pat in script.excludes]))
diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js
index 605f82d5a..eb2d8fea1 100644
--- a/qutebrowser/javascript/greasemonkey_wrapper.js
+++ b/qutebrowser/javascript/greasemonkey_wrapper.js
@@ -5,15 +5,19 @@
         console.log(text);
     }
 
-    var GM_info = (function () {
-        return {
-            'script': {{ scriptInfo | tojson }},
-            'scriptMetaStr': {{ scriptMeta | tojson }},
-            'scriptWillUpdate': false,
-            'version': '0.0.1',
-            'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction
-        };
-    }());
+    var GM_info = {
+        'script': {{ scriptInfo }},
+        'scriptMetaStr': {{ scriptMeta | tojson }},
+        'scriptWillUpdate': false,
+        'version': "0.0.1",
+        'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction
+    };
+
+    function checkKey(key, funcName) {
+        if (typeof key !== "string") {
+          throw new Error(funcName+" requires the first parameter to be of type string, not '"+typeof key+"'");
+        }
+    }
 
     function GM_setValue(key, value) {
         if (typeof key !== "string") {

From c0832eb04b87b6762f4d84440039bc6cf8dc472b Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Tue, 10 Oct 2017 20:52:41 +1300
Subject: [PATCH 20/35] Greasemonkey: support @nosubframes.

And run on frames by default. At least on webengine. There is probably
some api to enumerate frames on a webkit page.

Not tested.
---
 qutebrowser/browser/greasemonkey.py                | 5 +++++
 qutebrowser/browser/webengine/webenginesettings.py | 4 +++-
 qutebrowser/browser/webengine/webview.py           | 1 +
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index b58a558a5..cd6fce27a 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -49,6 +49,9 @@ class GreasemonkeyScript:
         self.name = None
         self.namespace = None
         self.run_at = None
+        self.script_meta = None
+        # Running on subframes is only supported on the qtwebengine backend.
+        self.runs_on_sub_frames = True
         for name, value in properties:
             if name == 'name':
                 self.name = value
@@ -62,6 +65,8 @@ class GreasemonkeyScript:
                 self.excludes.append(value)
             elif name == 'run-at':
                 self.run_at = value
+            elif name == 'noframes':
+                self.runs_on_sub_frames = False
 
     HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n'
     PROPS_REGEX = r'// @(?P<prop>[^\s]+)\s+(?P<val>.+)'
diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py
index 18ba98fd6..dade671c8 100644
--- a/qutebrowser/browser/webengine/webenginesettings.py
+++ b/qutebrowser/browser/webengine/webenginesettings.py
@@ -276,7 +276,9 @@ def inject_userscripts():
             new_script.setWorldId(QWebEngineScript.MainWorld)
             new_script.setSourceCode(script.code())
             new_script.setName("GM-{}".format(script.name))
-            log.greasemonkey.debug('adding script: %s', new_script.name())
+            new_script.setRunsOnSubFrames(script.runs_on_sub_frames)
+            log.greasemonkey.debug('adding script: {}'
+                                   .format(new_script.name()))
             scripts.insert(new_script)
 
 
diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py
index 35d6fca40..af4476d4a 100644
--- a/qutebrowser/browser/webengine/webview.py
+++ b/qutebrowser/browser/webengine/webview.py
@@ -329,6 +329,7 @@ class WebEnginePage(QWebEnginePage):
             new_script.setWorldId(QWebEngineScript.MainWorld)
             new_script.setSourceCode(script.code())
             new_script.setName("GM-{}".format(script.name))
+            new_script.setRunsOnSubFrames(script.runs_on_sub_frames)
             log.greasemonkey.debug("Adding script: {}"
                                    .format(new_script.name()))
             scripts.insert(new_script)

From 7c497427ce7cdaf3d23362cdae570b9660d7cf45 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Tue, 10 Oct 2017 20:54:58 +1300
Subject: [PATCH 21/35] Greasemonkey: various javascript fixups to GM wrapper
 template.

Thanks to @sandrosc. A few breaking changes fixed (default method to
GM_xhr not working, GM_listvalues not cleaning up output, GM_setvalue
param checking logic wrong) and a few hygenic changes made.
---
 .../javascript/greasemonkey_wrapper.js        | 36 ++++++++-----------
 1 file changed, 14 insertions(+), 22 deletions(-)

diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js
index eb2d8fea1..7a271375d 100644
--- a/qutebrowser/javascript/greasemonkey_wrapper.js
+++ b/qutebrowser/javascript/greasemonkey_wrapper.js
@@ -20,11 +20,9 @@
     }
 
     function GM_setValue(key, value) {
-        if (typeof key !== "string") {
-          throw new Error("GM_setValue requires the first parameter to be of type string, not '"+typeof key+"'");
-        }
-        if (typeof value !== "string" ||
-            typeof value !== "number" ||
+        checkKey(key, "GM_setValue");
+        if (typeof value !== "string" &&
+            typeof value !== "number" &&
             typeof value !== "boolean") {
           throw new Error("GM_setValue requires the second parameter to be of type string, number or boolean, not '"+typeof value+"'");
         }
@@ -32,24 +30,20 @@
     }
 
     function GM_getValue(key, default_) {
-        if (typeof key !== "string") {
-          throw new Error("GM_getValue requires the first parameter to be of type string, not '"+typeof key+"'");
-        }
+        checkKey(key, "GM_getValue");
         return localStorage.getItem(_qute_script_id + key) || default_;
     }
 
     function GM_deleteValue(key) {
-        if (typeof key !== "string") {
-          throw new Error("GM_deleteValue requires the first parameter to be of type string, not '"+typeof key+"'");
-        }
+        checkKey(key, "GM_deleteValue");
         localStorage.removeItem(_qute_script_id + key);
     }
 
     function GM_listValues() {
-        var i, keys = [];
-        for (i = 0; i < localStorage.length; i = i + 1) {
+        var keys = [];
+        for (var i = 0; i < localStorage.length; i++) {
             if (localStorage.key(i).startsWith(_qute_script_id)) {
-                keys.push(localStorage.key(i));
+                keys.push(localStorage.key(i).slice(_qute_script_id.length));
             }
         }
         return keys;
@@ -62,7 +56,7 @@
 
     // Almost verbatim copy from Eric
     function GM_xmlhttpRequest(/* object */ details) {
-        details.method = details.method.toUpperCase() || "GET";
+        details.method = details.method ? details.method.toUpperCase() : "GET";
 
         if (!details.url) {
             throw ("GM_xmlhttpRequest requires an URL.");
@@ -99,26 +93,24 @@
     }
 
     function GM_addStyle(/* String */ styles) {
+        var oStyle = document.createElement("style");
+        oStyle.setAttribute("type", "text/css");
+        oStyle.appendChild(document.createTextNode(styles));
+
         var head = document.getElementsByTagName("head")[0];
         if (head === undefined) {
             document.onreadystatechange = function () {
                 if (document.readyState == "interactive") {
-                    var oStyle = document.createElement("style");
-                    oStyle.setAttribute("type", "text/css");
-                    oStyle.appendChild(document.createTextNode(styles));
                     document.getElementsByTagName("head")[0].appendChild(oStyle);
                 }
             }
         }
         else {
-            var oStyle = document.createElement("style");
-            oStyle.setAttribute("type", "text/css");
-            oStyle.appendChild(document.createTextNode(styles));
             head.appendChild(oStyle);
         }
     }
 
-    unsafeWindow = window;
+    var unsafeWindow = window;
 
     //====== The actual user script source ======//
 {{ scriptSource }}

From 4c3461038dc4fe2a0732860f064f13a3d1c9f298 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Tue, 10 Oct 2017 20:59:27 +1300
Subject: [PATCH 22/35] Greasemonkey: add minimal end-to-end test.

Just runs a greasemonkey script on a test page and uses console.log to
ensure it is running.

Tests @include, and basic happy path greasemonkey.py operation (loading
and parsing script, scrip_for on webkit), only testing document-start
injecting point but that is the troublsome one at this point.

Tested on py35 debian unstable (oldwebkit and qtwebengine5.9) debian
stable qtwebengine5.7.

Note the extra :reload call for qt5.7 because document-start scripts
don't seem to run on the first page load with the current insertion
point. I need to look into this more to look at ways of fixing this.
---
 tests/end2end/features/javascript.feature     |  9 ++++++++
 tests/end2end/features/test_javascript_bdd.py | 22 +++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature
index a309d6187..66d108125 100644
--- a/tests/end2end/features/javascript.feature
+++ b/tests/end2end/features/javascript.feature
@@ -123,3 +123,12 @@ Feature: Javascript stuff
         And I wait for "[*/data/javascript/windowsize.html:*] loaded" in the log
         And I run :tab-next
         Then the window sizes should be the same
+
+    Scenario: Have a greasemonkey script run on a page
+        When I have a greasemonkey file saved
+        And I run :greasemonkey-reload
+        And I open data/title.html
+        # This second reload is required in webengine < 5.8 for scripts
+        # registered to run at document-start, some sort of timing issue.
+        And I run :reload
+        Then the javascript message "Script is running." should be logged
diff --git a/tests/end2end/features/test_javascript_bdd.py b/tests/end2end/features/test_javascript_bdd.py
index 9f6c021ce..cb81e1ef6 100644
--- a/tests/end2end/features/test_javascript_bdd.py
+++ b/tests/end2end/features/test_javascript_bdd.py
@@ -17,6 +17,8 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
+import os.path
+
 import pytest_bdd as bdd
 bdd.scenarios('javascript.feature')
 
@@ -29,3 +31,23 @@ def check_window_sizes(quteproc):
     hidden_size = hidden.message.split()[-1]
     visible_size = visible.message.split()[-1]
     assert hidden_size == visible_size
+
+test_gm_script="""
+// ==UserScript==
+// @name Qutebrowser test userscript
+// @namespace invalid.org
+// @include http://localhost:*/data/title.html
+// @exclude ???
+// @run-at document-start
+// ==/UserScript==
+console.log("Script is running on " + window.location.pathname);
+"""
+
+@bdd.when("I have a greasemonkey file saved")
+def create_greasemonkey_file(quteproc):
+    script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey')
+    os.mkdir(script_path)
+    file_path = os.path.join(script_path, 'test.user.js')
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(test_gm_script)
+

From 9aeb5775c1856a612d72e4bb2e791d3a0c50775c Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Tue, 24 Oct 2017 21:19:22 +1300
Subject: [PATCH 23/35] greasemonkey: run scripts on subframes on webkit

Use the `QWebPage.frameCreated` signal to get notifications of subframes
and connect the javascript injection triggering signals on those frames
too.

I had to add a `url = url() or requestedUrl()` bit in there because the
inject_userjs method was getting called to early or something when
frame.url() wasn't set or was set to the previous page so we were
passing the wrong url to greasemonkey.scripts_for().

I ran into a bizarre (I maybe it is completely obvious and I just don't
see it) issue where the signals attached to the main frame that were
connected to a partial function with the main frame as an argument were
not getting emitted, or at least those partial functions were not being
called. I worked around it by using None to mean defaulting to the main
frame in a couple of places.
---
 qutebrowser/browser/greasemonkey.py   |  1 -
 qutebrowser/browser/webkit/webpage.py | 63 +++++++++++++++++++++------
 2 files changed, 49 insertions(+), 15 deletions(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index cd6fce27a..c8ada7b9a 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -50,7 +50,6 @@ class GreasemonkeyScript:
         self.namespace = None
         self.run_at = None
         self.script_meta = None
-        # Running on subframes is only supported on the qtwebengine backend.
         self.runs_on_sub_frames = True
         for name, value in properties:
             if name == 'name':
diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index e48519189..299815cc3 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -86,12 +86,33 @@ class BrowserPage(QWebPage):
             self.on_save_frame_state_requested)
         self.restoreFrameStateRequested.connect(
             self.on_restore_frame_state_requested)
-        self.mainFrame().javaScriptWindowObjectCleared.connect(
-            functools.partial(self.inject_userjs, load='start'))
-        self.mainFrame().initialLayoutCompleted.connect(
-            functools.partial(self.inject_userjs, load='end'))
-        self.mainFrame().loadFinished.connect(
-            functools.partial(self.inject_userjs, load='idle'))
+        self.connect_userjs_signals(None)
+        self.frameCreated.connect(self.connect_userjs_signals)
+
+    @pyqtSlot('QWebFrame*')
+    def connect_userjs_signals(self, frame_arg):
+        """
+        Connect the signals used as triggers for injecting user
+        javascripts into `frame_arg`.
+        """
+        # If we pass whatever self.mainFrame() or self.currentFrame() returns
+        # at init time into the partial functions which the signals
+        # below call then the signals don't seem to be called at all for
+        # the main frame of the first tab. I have no idea why I am
+        # seeing this behavior. Replace the None in the call to this
+        # function in __init__ with self.mainFrame() and try for
+        # yourself.
+        if frame_arg:
+            frame = frame_arg
+        else:
+            frame = self.mainFrame()
+
+        frame.javaScriptWindowObjectCleared.connect(
+            functools.partial(self.inject_userjs, frame_arg, load='start'))
+        frame.initialLayoutCompleted.connect(
+            functools.partial(self.inject_userjs, frame_arg, load='end'))
+        frame.loadFinished.connect(
+            functools.partial(self.inject_userjs, frame_arg, load='idle'))
 
     def javaScriptPrompt(self, frame, js_msg, default):
         """Override javaScriptPrompt to use qutebrowser prompts."""
@@ -290,16 +311,24 @@ class BrowserPage(QWebPage):
             self.error_occurred = False
 
     @pyqtSlot()
-    def inject_userjs(self, load):
+    def inject_userjs(self, frame, load):
         """Inject user javascripts into the page.
 
-        param: The page load stage to inject the corresponding scripts
-               for. Support values are "start", "end" and "idle",
-               corresponding to the allowed values of the `@run-at`
-               directive in the greasemonkey metadata spec.
+        Args:
+            frame: The QWebFrame to inject the user scripts into, or
+                   None for the main frame.
+            load: The page load stage to inject the corresponding
+                  scripts for. Support values are "start", "end" and
+                  "idle", corresponding to the allowed values of the
+                  `@run-at` directive in the greasemonkey metadata spec.
         """
+        if not frame:
+            frame = self.mainFrame()
+        url = frame.url()
+        if url.isEmpty():
+            url = frame.requestedUrl()
+
         greasemonkey = objreg.get('greasemonkey')
-        url = self.currentFrame().url()
         scripts = greasemonkey.scripts_for(url)
 
         if load == "start":
@@ -309,9 +338,15 @@ class BrowserPage(QWebPage):
         elif load == "idle":
             toload = scripts.idle
 
+        if url.isEmpty():
+            # This happens during normal usage like with view source but may
+            # also indicate a bug.
+            log.greasemonkey.debug("Not running scripts for frame with no "
+                                   "url: {}".format(frame))
         for script in toload:
-            log.webview.debug('Running GM script: {}'.format(script.name))
-            self.currentFrame().evaluateJavaScript(script.code())
+            if frame is self.mainFrame() or script.runs_on_sub_frames:
+                log.webview.debug('Running GM script: {}'.format(script.name))
+                frame.evaluateJavaScript(script.code())
 
     @pyqtSlot('QWebFrame*', 'QWebPage::Feature')
     def _on_feature_permission_requested(self, frame, feature):

From 361a1ed6e46bf602825986e97c540b17c44b12a7 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 1 Nov 2017 23:37:53 +1300
Subject: [PATCH 24/35] Greasemonkey: change PROPS_REGEX to handle non-value
 keys.

We weren't actually picking up the @noframes greasemonkey directive
because of this. I haven't tested this very extensively but it seems to
work for making the property value optional.
---
 qutebrowser/browser/greasemonkey.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index c8ada7b9a..845b83d85 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -68,7 +68,7 @@ class GreasemonkeyScript:
                 self.runs_on_sub_frames = False
 
     HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n'
-    PROPS_REGEX = r'// @(?P<prop>[^\s]+)\s+(?P<val>.+)'
+    PROPS_REGEX = r'// @(?P<prop>[^\s]+)\s*(?P<val>.*)'
 
     @classmethod
     def parse(cls, source):

From dd59f8d724ccd3c3e86b201511ceb07da483a53e Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 1 Nov 2017 23:41:52 +1300
Subject: [PATCH 25/35] Greasemonkey: add more end2end tests

Test document-end and noframes. Because coverage.py told me to.
Hopefully this doesn't slow the test run down too much, particularly the
"should not be logged" bit.

I'm just reusing and existing test html page that used an iframe because
I'm lazy.
---
 tests/end2end/features/javascript.feature     | 20 ++++++++---
 tests/end2end/features/test_javascript_bdd.py | 35 +++++++++++++------
 2 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature
index 66d108125..aaad84be3 100644
--- a/tests/end2end/features/javascript.feature
+++ b/tests/end2end/features/javascript.feature
@@ -124,11 +124,23 @@ Feature: Javascript stuff
         And I run :tab-next
         Then the window sizes should be the same
 
-    Scenario: Have a greasemonkey script run on a page
-        When I have a greasemonkey file saved
+    Scenario: Have a greasemonkey script run at page start
+        When I have a greasemonkey file saved for document-start with noframes unset
         And I run :greasemonkey-reload
-        And I open data/title.html
+        And I open data/hints/iframe.html
         # This second reload is required in webengine < 5.8 for scripts
         # registered to run at document-start, some sort of timing issue.
         And I run :reload
-        Then the javascript message "Script is running." should be logged
+        Then the javascript message "Script is running on /data/hints/iframe.html" should be logged
+
+    Scenario: Have a greasemonkey script running on frames
+        When I have a greasemonkey file saved for document-end with noframes unset
+        And I run :greasemonkey-reload
+        And I open data/hints/iframe.html
+        Then the javascript message "Script is running on /data/hints/html/wrapped.html" should be logged
+
+    Scenario: Have a greasemonkey script running on noframes
+        When I have a greasemonkey file saved for document-end with noframes set
+        And I run :greasemonkey-reload
+        And I open data/hints/iframe.html
+        Then the javascript message "Script is running on /data/hints/html/wrapped.html" should not be logged
diff --git a/tests/end2end/features/test_javascript_bdd.py b/tests/end2end/features/test_javascript_bdd.py
index cb81e1ef6..16896d4b5 100644
--- a/tests/end2end/features/test_javascript_bdd.py
+++ b/tests/end2end/features/test_javascript_bdd.py
@@ -32,22 +32,37 @@ def check_window_sizes(quteproc):
     visible_size = visible.message.split()[-1]
     assert hidden_size == visible_size
 
-test_gm_script="""
+
+test_gm_script = r"""
 // ==UserScript==
 // @name Qutebrowser test userscript
 // @namespace invalid.org
-// @include http://localhost:*/data/title.html
+// @include http://localhost:*/data/hints/iframe.html
+// @include http://localhost:*/data/hints/html/wrapped.html
 // @exclude ???
-// @run-at document-start
+// @run-at {stage}
+// {frames}
 // ==/UserScript==
 console.log("Script is running on " + window.location.pathname);
 """
 
-@bdd.when("I have a greasemonkey file saved")
-def create_greasemonkey_file(quteproc):
-    script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey')
-    os.mkdir(script_path)
-    file_path = os.path.join(script_path, 'test.user.js')
-    with open(file_path, 'w', encoding='utf-8') as f:
-        f.write(test_gm_script)
 
+@bdd.when(bdd.parsers.parse("I have a greasemonkey file saved for {stage} "
+                            "with noframes {frameset}"))
+def create_greasemonkey_file(quteproc, stage, frameset):
+    script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey')
+    try:
+        os.mkdir(script_path)
+    except FileExistsError:
+        pass
+    file_path = os.path.join(script_path, 'test.user.js')
+    if frameset == "set":
+        frames = "@noframes"
+    elif frameset == "unset":
+        frames = ""
+    else:
+        raise ValueError("noframes can only be set or unset, "
+                         "not {}".format(frameset))
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(test_gm_script.format(stage=stage,
+                                     frames=frames))

From 92b48e77c79cf5b1206fdd953a92678315f083e6 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 1 Nov 2017 23:45:31 +1300
Subject: [PATCH 26/35] Greasemonkey: add unit tests for GreasemonkeyManager

---
 tests/unit/javascript/test_greasemonkey.py | 108 +++++++++++++++++++++
 1 file changed, 108 insertions(+)
 create mode 100644 tests/unit/javascript/test_greasemonkey.py

diff --git a/tests/unit/javascript/test_greasemonkey.py b/tests/unit/javascript/test_greasemonkey.py
new file mode 100644
index 000000000..b0ba64bdf
--- /dev/null
+++ b/tests/unit/javascript/test_greasemonkey.py
@@ -0,0 +1,108 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+# Copyright 2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Tests for qutebrowser.browser.greasemonkey."""
+
+import os
+import logging
+
+import pytest
+from PyQt5.QtCore import QUrl
+
+from qutebrowser.browser import greasemonkey
+
+test_gm_script = """
+// ==UserScript==
+// @name Qutebrowser test userscript
+// @namespace invalid.org
+// @include http://localhost:*/data/title.html
+// @match http://trolol*
+// @exclude https://badhost.xxx/*
+// @run-at document-start
+// ==/UserScript==
+console.log("Script is running.");
+"""
+
+pytestmark = pytest.mark.usefixtures('data_tmpdir')
+
+
+def save_script(script_text, filename):
+    script_path = greasemonkey._scripts_dir()
+    try:
+        os.mkdir(script_path)
+    except FileExistsError:
+        pass
+    file_path = os.path.join(script_path, filename)
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(script_text)
+
+
+def test_all():
+    """Test that a script gets read from file, parsed and returned."""
+    save_script(test_gm_script, 'test.user.js')
+
+    gm_manager = greasemonkey.GreasemonkeyManager()
+    assert (gm_manager.all_scripts()[0].name ==
+            "Qutebrowser test userscript")
+
+
+@pytest.mark.parametrize("url, expected_matches", [
+    # included
+    ('http://trololololololo.com/', 1),
+    # neither included nor excluded
+    ('http://aaaaaaaaaa.com/', 0),
+    # excluded
+    ('https://badhost.xxx/', 0),
+])
+def test_get_scripts_by_url(url, expected_matches):
+    """Check greasemonkey include/exclude rules work."""
+    save_script(test_gm_script, 'test.user.js')
+    gm_manager = greasemonkey.GreasemonkeyManager()
+
+    scripts = gm_manager.scripts_for(QUrl(url))
+    assert (len(scripts.start + scripts.end + scripts.idle) ==
+            expected_matches)
+
+
+def test_no_metadata(caplog):
+    """Run on all sites at document-end is the default."""
+    save_script("var nothing = true;\n", 'nothing.user.js')
+
+    with caplog.at_level(logging.WARNING):
+        gm_manager = greasemonkey.GreasemonkeyManager()
+
+    scripts = gm_manager.scripts_for(QUrl('http://notamatch.invalid/'))
+    assert len(scripts.start + scripts.end + scripts.idle) == 1
+    assert len(scripts.end) == 1
+
+
+def test_bad_scheme(caplog):
+    """qute:// isn't in the list of allowed schemes."""
+    save_script("var nothing = true;\n", 'nothing.user.js')
+
+    with caplog.at_level(logging.WARNING):
+        gm_manager = greasemonkey.GreasemonkeyManager()
+
+    scripts = gm_manager.scripts_for(QUrl('qute://settings'))
+    assert len(scripts.start + scripts.end + scripts.idle) == 0
+
+
+def test_load_emits_signal(qtbot):
+    gm_manager = greasemonkey.GreasemonkeyManager()
+    with qtbot.wait_signal(gm_manager.scripts_reloaded):
+        gm_manager.load_scripts()

From 8a5b42ffbd6ad878fa5810819086b1db77fe8842 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Sat, 4 Nov 2017 16:11:58 +1300
Subject: [PATCH 27/35] Greasemonkey: es6ify the greasemonkey wrapper js.

Because backwards compatibility sucks I guess.
---
 .../javascript/greasemonkey_wrapper.js        | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js
index 7a271375d..e86991040 100644
--- a/qutebrowser/javascript/greasemonkey_wrapper.js
+++ b/qutebrowser/javascript/greasemonkey_wrapper.js
@@ -1,11 +1,11 @@
 (function () {
-    var _qute_script_id = "__gm_"+{{ scriptName | tojson }};
+    const _qute_script_id = "__gm_"+{{ scriptName | tojson }};
 
     function GM_log(text) {
         console.log(text);
     }
 
-    var GM_info = {
+    const GM_info = {
         'script': {{ scriptInfo }},
         'scriptMetaStr': {{ scriptMeta | tojson }},
         'scriptWillUpdate': false,
@@ -15,7 +15,7 @@
 
     function checkKey(key, funcName) {
         if (typeof key !== "string") {
-          throw new Error(funcName+" requires the first parameter to be of type string, not '"+typeof key+"'");
+          throw new Error(`${funcName} requires the first parameter to be of type string, not '${typeof key}'`);
         }
     }
 
@@ -24,7 +24,7 @@
         if (typeof value !== "string" &&
             typeof value !== "number" &&
             typeof value !== "boolean") {
-          throw new Error("GM_setValue requires the second parameter to be of type string, number or boolean, not '"+typeof value+"'");
+          throw new Error(`GM_setValue requires the second parameter to be of type string, number or boolean, not '${typeof value}'`);
         }
         localStorage.setItem(_qute_script_id + key, value);
     }
@@ -40,8 +40,8 @@
     }
 
     function GM_listValues() {
-        var keys = [];
-        for (var i = 0; i < localStorage.length; i++) {
+        let keys = [];
+        for (let i = 0; i < localStorage.length; i++) {
             if (localStorage.key(i).startsWith(_qute_script_id)) {
                 keys.push(localStorage.key(i).slice(_qute_script_id.length));
             }
@@ -63,7 +63,7 @@
         }
 
         // build XMLHttpRequest object
-        var oXhr = new XMLHttpRequest();
+        let oXhr = new XMLHttpRequest();
         // run it
         if ("onreadystatechange" in details) {
             oXhr.onreadystatechange = function () {
@@ -80,7 +80,7 @@
         oXhr.open(details.method, details.url, true);
 
         if ("headers" in details) {
-            for (var header in details.headers) {
+            for (let header in details.headers) {
                 oXhr.setRequestHeader(header, details.headers[header]);
             }
         }
@@ -93,11 +93,11 @@
     }
 
     function GM_addStyle(/* String */ styles) {
-        var oStyle = document.createElement("style");
+        let oStyle = document.createElement("style");
         oStyle.setAttribute("type", "text/css");
         oStyle.appendChild(document.createTextNode(styles));
 
-        var head = document.getElementsByTagName("head")[0];
+        let head = document.getElementsByTagName("head")[0];
         if (head === undefined) {
             document.onreadystatechange = function () {
                 if (document.readyState == "interactive") {
@@ -110,7 +110,7 @@
         }
     }
 
-    var unsafeWindow = window;
+    const unsafeWindow = window;
 
     //====== The actual user script source ======//
 {{ scriptSource }}

From df624944f99a0dca33900ddaed2cd49f73880ac6 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 8 Nov 2017 21:03:58 +1300
Subject: [PATCH 28/35] Greasemonkey: webkit: injected all scripts on
 loadFinished.

The signal we were using to inject greasemonkey scripts registered to
run at document-start (javaScriptWindowObjectCleared) was unreliable to
non-existant. The initialLayoutCompleted signal is a bit of an odd duck
too I suppose. Anyway, we don't anticipate any scripts would break from
being injected when the page is finished loaded that wouldn't already
have been flaky due to the complexities of the modern web. If there is
an issue hopefully someone raises an issue and we can look into it.
---
 qutebrowser/browser/webkit/webpage.py | 51 +++++++--------------------
 1 file changed, 13 insertions(+), 38 deletions(-)

diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index 299815cc3..f9bd51194 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -86,33 +86,18 @@ class BrowserPage(QWebPage):
             self.on_save_frame_state_requested)
         self.restoreFrameStateRequested.connect(
             self.on_restore_frame_state_requested)
-        self.connect_userjs_signals(None)
+        self.connect_userjs_signals(self.mainFrame())
         self.frameCreated.connect(self.connect_userjs_signals)
 
     @pyqtSlot('QWebFrame*')
-    def connect_userjs_signals(self, frame_arg):
-        """
-        Connect the signals used as triggers for injecting user
-        javascripts into `frame_arg`.
-        """
-        # If we pass whatever self.mainFrame() or self.currentFrame() returns
-        # at init time into the partial functions which the signals
-        # below call then the signals don't seem to be called at all for
-        # the main frame of the first tab. I have no idea why I am
-        # seeing this behavior. Replace the None in the call to this
-        # function in __init__ with self.mainFrame() and try for
-        # yourself.
-        if frame_arg:
-            frame = frame_arg
-        else:
-            frame = self.mainFrame()
+    def connect_userjs_signals(self, frame):
+        """Connect userjs related signals to `frame`.
 
-        frame.javaScriptWindowObjectCleared.connect(
-            functools.partial(self.inject_userjs, frame_arg, load='start'))
-        frame.initialLayoutCompleted.connect(
-            functools.partial(self.inject_userjs, frame_arg, load='end'))
+        Connect the signals used as triggers for injecting user
+        javascripts into the passed QWebFrame.
+        """
         frame.loadFinished.connect(
-            functools.partial(self.inject_userjs, frame_arg, load='idle'))
+            functools.partial(self.inject_userjs, frame))
 
     def javaScriptPrompt(self, frame, js_msg, default):
         """Override javaScriptPrompt to use qutebrowser prompts."""
@@ -311,32 +296,22 @@ class BrowserPage(QWebPage):
             self.error_occurred = False
 
     @pyqtSlot()
-    def inject_userjs(self, frame, load):
+    def inject_userjs(self, frame):
         """Inject user javascripts into the page.
 
         Args:
-            frame: The QWebFrame to inject the user scripts into, or
-                   None for the main frame.
-            load: The page load stage to inject the corresponding
-                  scripts for. Support values are "start", "end" and
-                  "idle", corresponding to the allowed values of the
-                  `@run-at` directive in the greasemonkey metadata spec.
+            frame: The QWebFrame to inject the user scripts into.
         """
-        if not frame:
-            frame = self.mainFrame()
         url = frame.url()
         if url.isEmpty():
             url = frame.requestedUrl()
 
         greasemonkey = objreg.get('greasemonkey')
         scripts = greasemonkey.scripts_for(url)
-
-        if load == "start":
-            toload = scripts.start
-        elif load == "end":
-            toload = scripts.end
-        elif load == "idle":
-            toload = scripts.idle
+        # QtWebKit has trouble providing us with signals representing
+        # page load progress at reasonable times, so we just load all
+        # scripts on the same event.
+        toload = scripts.start + scripts.end + scripts.idle
 
         if url.isEmpty():
             # This happens during normal usage like with view source but may

From 6933bc05b4ba93ef6d8d72ec582362a7db36aef5 Mon Sep 17 00:00:00 2001
From: Florian Bruhin <git@the-compiler.org>
Date: Mon, 20 Nov 2017 07:31:01 +0100
Subject: [PATCH 29/35] Add some debug logging for GreaseMonkey with QtWebKit

---
 qutebrowser/browser/webkit/webpage.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index f9bd51194..7c67ab64b 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -96,6 +96,8 @@ class BrowserPage(QWebPage):
         Connect the signals used as triggers for injecting user
         javascripts into the passed QWebFrame.
         """
+        log.greasemonkey.debug("Connecting to frame {} ({})"
+                               .format(frame, frame.url().toDisplayString()))
         frame.loadFinished.connect(
             functools.partial(self.inject_userjs, frame))
 
@@ -306,6 +308,9 @@ class BrowserPage(QWebPage):
         if url.isEmpty():
             url = frame.requestedUrl()
 
+        log.greasemonkey.debug("inject_userjs called for {} ({})"
+                               .format(frame, url.toDisplayString()))
+
         greasemonkey = objreg.get('greasemonkey')
         scripts = greasemonkey.scripts_for(url)
         # QtWebKit has trouble providing us with signals representing

From db353c40300ef8d4066026b5650c27daa2da61e3 Mon Sep 17 00:00:00 2001
From: Florian Bruhin <git@the-compiler.org>
Date: Mon, 20 Nov 2017 07:31:13 +0100
Subject: [PATCH 30/35] Connect the page signal for GreaseMonkey

Looks like we don't get the mainFrame's loadFinished signal properly.
---
 qutebrowser/browser/webkit/webpage.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index 7c67ab64b..24a213a42 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -86,7 +86,8 @@ class BrowserPage(QWebPage):
             self.on_save_frame_state_requested)
         self.restoreFrameStateRequested.connect(
             self.on_restore_frame_state_requested)
-        self.connect_userjs_signals(self.mainFrame())
+        self.loadFinished.connect(
+            functools.partial(self.inject_userjs, self.mainFrame()))
         self.frameCreated.connect(self.connect_userjs_signals)
 
     @pyqtSlot('QWebFrame*')

From 0e80be2d30ac90b82db968d077d498ad9c1e8f97 Mon Sep 17 00:00:00 2001
From: Florian Bruhin <git@the-compiler.org>
Date: Mon, 20 Nov 2017 08:31:10 +0100
Subject: [PATCH 31/35] Clear end2end test data again after initializing

If we don't do this, earlier tests can affect later ones when e.g. using "...
should not be logged", as we don't really wait until a test has been fully
finished.
---
 tests/end2end/fixtures/quteprocess.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py
index 27c347ca4..3164f5fde 100644
--- a/tests/end2end/fixtures/quteprocess.py
+++ b/tests/end2end/fixtures/quteprocess.py
@@ -527,6 +527,7 @@ class QuteProc(testprocess.Process):
         super().before_test()
         self.send_cmd(':config-clear')
         self._init_settings()
+        self.clear_data()
 
     def _init_settings(self):
         """Adjust some qutebrowser settings after starting."""

From 0381d74e9aac2d045567db13ffc2ccca55d57160 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Mon, 27 Nov 2017 20:06:29 +1300
Subject: [PATCH 32/35] Greasemonkey: privatise some utility functions

---
 qutebrowser/browser/webkit/webpage.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index 24a213a42..89b293869 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -87,11 +87,11 @@ class BrowserPage(QWebPage):
         self.restoreFrameStateRequested.connect(
             self.on_restore_frame_state_requested)
         self.loadFinished.connect(
-            functools.partial(self.inject_userjs, self.mainFrame()))
-        self.frameCreated.connect(self.connect_userjs_signals)
+            functools.partial(self._inject_userjs, self.mainFrame()))
+        self.frameCreated.connect(self._connect_userjs_signals)
 
     @pyqtSlot('QWebFrame*')
-    def connect_userjs_signals(self, frame):
+    def _connect_userjs_signals(self, frame):
         """Connect userjs related signals to `frame`.
 
         Connect the signals used as triggers for injecting user
@@ -100,7 +100,7 @@ class BrowserPage(QWebPage):
         log.greasemonkey.debug("Connecting to frame {} ({})"
                                .format(frame, frame.url().toDisplayString()))
         frame.loadFinished.connect(
-            functools.partial(self.inject_userjs, frame))
+            functools.partial(self._inject_userjs, frame))
 
     def javaScriptPrompt(self, frame, js_msg, default):
         """Override javaScriptPrompt to use qutebrowser prompts."""
@@ -298,8 +298,7 @@ class BrowserPage(QWebPage):
         else:
             self.error_occurred = False
 
-    @pyqtSlot()
-    def inject_userjs(self, frame):
+    def _inject_userjs(self, frame):
         """Inject user javascripts into the page.
 
         Args:
@@ -309,7 +308,7 @@ class BrowserPage(QWebPage):
         if url.isEmpty():
             url = frame.requestedUrl()
 
-        log.greasemonkey.debug("inject_userjs called for {} ({})"
+        log.greasemonkey.debug("_inject_userjs called for {} ({})"
                                .format(frame, url.toDisplayString()))
 
         greasemonkey = objreg.get('greasemonkey')

From 129f97873a57d4b56b9c399e9da5c9da70dd880c Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Mon, 27 Nov 2017 20:07:16 +1300
Subject: [PATCH 33/35] Greasemonkey: add assert to tests scripts_for
 assumptions.

And crash the users browsing session as a result of any accidental and
totally, otherwise, non-fatal unforseen errors.
---
 qutebrowser/browser/webkit/webpage.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py
index 89b293869..02aa270d7 100644
--- a/qutebrowser/browser/webkit/webpage.py
+++ b/qutebrowser/browser/webkit/webpage.py
@@ -323,6 +323,8 @@ class BrowserPage(QWebPage):
             # also indicate a bug.
             log.greasemonkey.debug("Not running scripts for frame with no "
                                    "url: {}".format(frame))
+            assert not toload, toload
+
         for script in toload:
             if frame is self.mainFrame() or script.runs_on_sub_frames:
                 log.webview.debug('Running GM script: {}'.format(script.name))

From ead108eeebadfc6e73b5927901eb155eebad028b Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 6 Dec 2017 20:27:56 +1300
Subject: [PATCH 34/35] fixup! Greasemonkey: Add run-at document-idle.

---
 qutebrowser/browser/greasemonkey.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py
index 845b83d85..3fd01137f 100644
--- a/qutebrowser/browser/greasemonkey.py
+++ b/qutebrowser/browser/greasemonkey.py
@@ -193,8 +193,8 @@ class GreasemonkeyManager(QObject):
         match = functools.partial(fnmatch.fnmatch,
                                   url.toString(QUrl.FullyEncoded))
         tester = (lambda script:
-                  any([match(pat) for pat in script.includes]) and
-                  not any([match(pat) for pat in script.excludes]))
+                  any(match(pat) for pat in script.includes) and
+                  not any(match(pat) for pat in script.excludes))
         return MatchingScripts(
             url,
             [script for script in self._run_start if tester(script)],

From 6b3e16b1630acfc8a78f4a8994d9ad2279450386 Mon Sep 17 00:00:00 2001
From: Jimmy <jimmy@spalge.com>
Date: Wed, 6 Dec 2017 20:34:29 +1300
Subject: [PATCH 35/35] Greasemonkey: mark failing no(sub)frames test as flaky.

This test is supposed to ensure that user scripts don't run on iframes
when the @noframes directive is set in the greasemonkey metadata. It is
failing sometimes on travis but passing on local test runs. Personally I
haven't actually ran the whole test suite through, just the javascript
tests. It maybe be some stale state that only shows up when you run the
whole suite. It may be some timing issue that only shows up on travis
because ???. Hopefully this stops the red x from showing up on the PR.
---
 tests/end2end/features/javascript.feature | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature
index aaad84be3..944d2606d 100644
--- a/tests/end2end/features/javascript.feature
+++ b/tests/end2end/features/javascript.feature
@@ -139,6 +139,7 @@ Feature: Javascript stuff
         And I open data/hints/iframe.html
         Then the javascript message "Script is running on /data/hints/html/wrapped.html" should be logged
 
+    @flaky
     Scenario: Have a greasemonkey script running on noframes
         When I have a greasemonkey file saved for document-end with noframes set
         And I run :greasemonkey-reload