diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py index 4f390b18b..dd112e00a 100644 --- a/qutebrowser/browser/downloads.py +++ b/qutebrowser/browser/downloads.py @@ -238,11 +238,14 @@ class FileDownloadTarget(_DownloadTarget): Attributes: filename: Filename where the download should be saved. + force_overwrite: Whether to overwrite the target without + prompting the user. """ - def __init__(self, filename): + def __init__(self, filename, force_overwrite=False): # pylint: disable=super-init-not-called self.filename = filename + self.force_overwrite = force_overwrite def suggested_filename(self): return os.path.basename(self.filename) @@ -738,7 +741,8 @@ class AbstractDownloadItem(QObject): if isinstance(target, FileObjDownloadTarget): self._set_fileobj(target.fileobj, autoclose=False) elif isinstance(target, FileDownloadTarget): - self._set_filename(target.filename) + self._set_filename( + target.filename, force_overwrite=target.force_overwrite) elif isinstance(target, OpenFileDownloadTarget): try: fobj = temp_download_manager.get_tmpfile(self.basename) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index fb064f6c1..6879f4cf6 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -23,13 +23,16 @@ import re import os import json import fnmatch +import functools import glob +import textwrap import attr from PyQt5.QtCore import pyqtSignal, QObject, QUrl -from qutebrowser.utils import log, standarddir, jinja, objreg +from qutebrowser.utils import log, standarddir, jinja, objreg, utils from qutebrowser.commands import cmdutils +from qutebrowser.browser import downloads def _scripts_dir(): @@ -45,6 +48,7 @@ class GreasemonkeyScript: self._code = code self.includes = [] self.excludes = [] + self.requires = [] self.description = None self.name = None self.namespace = None @@ -66,6 +70,8 @@ class GreasemonkeyScript: self.run_at = value elif name == 'noframes': self.runs_on_sub_frames = False + elif name == 'require': + self.requires.append(value) HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n' PROPS_REGEX = r'// @(?P[^\s]+)\s*(?P.*)' @@ -93,7 +99,7 @@ class GreasemonkeyScript: """Return the processed JavaScript code of this script. Adorns the source code with GM_* methods for Greasemonkey - compatibility and wraps it in an IFFE to hide it within a + compatibility and wraps it in an IIFE to hide it within a lexical scope. Note that this means line numbers in your browser's debugger/inspector will not match up to the line numbers in the source script directly. @@ -115,6 +121,14 @@ class GreasemonkeyScript: 'run-at': self.run_at, }) + def add_required_script(self, source): + """Add the source of a required script to this script.""" + # The additional source is indented in case it also contains a + # metadata block. Because we pass everything at once to + # QWebEngineScript and that would parse the first metadata block + # found as the valid one. + self._code = "\n".join([textwrap.indent(source, " "), self._code]) + @attr.s class MatchingScripts(object): @@ -145,15 +159,24 @@ class GreasemonkeyManager(QObject): def __init__(self, parent=None): super().__init__(parent) + self._run_start = [] + self._run_end = [] + self._run_idle = [] + self._in_progress_dls = [] + self.load_scripts() @cmdutils.register(name='greasemonkey-reload', instance='greasemonkey') - def load_scripts(self): + def load_scripts(self, force=False): """Re-read Greasemonkey scripts from disk. The scripts are read from a 'greasemonkey' subdirectory in qutebrowser's data directory (see `:version`). + + Args: + force: For any scripts that have required dependencies, + re-download them. """ self._run_start = [] self._run_end = [] @@ -169,24 +192,115 @@ class GreasemonkeyManager(QObject): script = GreasemonkeyScript.parse(script_file.read()) if not script.name: script.name = script_filename - - if script.run_at == 'document-start': - self._run_start.append(script) - elif script.run_at == 'document-end': - self._run_end.append(script) - elif script.run_at == 'document-idle': - self._run_idle.append(script) - else: - if script.run_at: - log.greasemonkey.warning( - "Script {} has invalid run-at defined, " - "defaulting to document-end".format(script_path)) - # Default as per - # https://wiki.greasespot.net/Metadata_Block#.40run-at - self._run_end.append(script) - log.greasemonkey.debug("Loaded script: {}".format(script.name)) + self.add_script(script, force) self.scripts_reloaded.emit() + def add_script(self, script, force=False): + """Add a GreasemonkeyScript to this manager. + + Args: + force: Fetch and overwrite any dependancies which are + already locally cached. + """ + if script.requires: + log.greasemonkey.debug( + "Deferring script until requirements are " + "fulfilled: {}".format(script.name)) + self._get_required_scripts(script, force) + else: + self._add_script(script) + + def _add_script(self, script): + if script.run_at == 'document-start': + self._run_start.append(script) + elif script.run_at == 'document-end': + self._run_end.append(script) + elif script.run_at == 'document-idle': + self._run_idle.append(script) + else: + if script.run_at: + log.greasemonkey.warning("Script {} has invalid run-at " + "defined, defaulting to " + "document-end" + .format(script.name)) + # Default as per + # https://wiki.greasespot.net/Metadata_Block#.40run-at + self._run_end.append(script) + log.greasemonkey.debug("Loaded script: {}".format(script.name)) + + def _required_url_to_file_path(self, url): + requires_dir = os.path.join(_scripts_dir(), 'requires') + if not os.path.exists(requires_dir): + os.mkdir(requires_dir) + return os.path.join(requires_dir, utils.sanitize_filename(url)) + + def _on_required_download_finished(self, script, download): + self._in_progress_dls.remove(download) + if not self._add_script_with_requires(script): + log.greasemonkey.debug( + "Finished download {} for script {} " + "but some requirements are still pending" + .format(download.basename, script.name)) + + def _add_script_with_requires(self, script, quiet=False): + """Add a script with pending downloads to this GreasemonkeyManager. + + Specifically a script that has dependancies specified via an + `@require` rule. + + Args: + script: The GreasemonkeyScript to add. + quiet: True to suppress the scripts_reloaded signal after + adding `script`. + Returns: True if the script was added, False if there are still + dependancies being downloaded. + """ + # See if we are still waiting on any required scripts for this one + for dl in self._in_progress_dls: + if dl.requested_url in script.requires: + return False + + # Need to add the required scripts to the IIFE now + for url in reversed(script.requires): + target_path = self._required_url_to_file_path(url) + log.greasemonkey.debug( + "Adding required script for {} to IIFE: {}" + .format(script.name, url)) + with open(target_path, encoding='utf8') as f: + script.add_required_script(f.read()) + + self._add_script(script) + if not quiet: + self.scripts_reloaded.emit() + return True + + def _get_required_scripts(self, script, force=False): + required_dls = [(url, self._required_url_to_file_path(url)) + for url in script.requires] + if not force: + required_dls = [(url, path) for (url, path) in required_dls + if not os.path.exists(path)] + if not required_dls: + # All the required files exist already + self._add_script_with_requires(script, quiet=True) + return + + download_manager = objreg.get('qtnetwork-download-manager') + + for url, target_path in required_dls: + target = downloads.FileDownloadTarget(target_path, + force_overwrite=True) + download = download_manager.get(QUrl(url), target=target, + auto_remove=True) + download.requested_url = url + self._in_progress_dls.append(download) + if download.successful: + self._on_required_download_finished(script, download) + else: + download.finished.connect( + functools.partial(self._on_required_download_finished, + script, download)) + def scripts_for(self, url): """Fetch scripts that are registered to run for url. diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index 2d36220dc..71266755a 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -110,6 +110,44 @@ } } + // Stub these two so that the gm4 polyfill script doesn't try to + // create broken versions as attributes of window. + function GM_getResourceText(caption, commandFunc, accessKey) { + console.error(`${GM_info.script.name} called unimplemented GM_getResourceText`); + } + + function GM_registerMenuCommand(caption, commandFunc, accessKey) { + console.error(`${GM_info.script.name} called unimplemented GM_registerMenuCommand`); + } + + // Mock the greasemonkey 4.0 async API. + const GM = {}; + GM.info = GM_info; + const entries = { + 'log': GM_log, + 'addStyle': GM_addStyle, + 'deleteValue': GM_deleteValue, + 'getValue': GM_getValue, + 'listValues': GM_listValues, + 'openInTab': GM_openInTab, + 'setValue': GM_setValue, + 'xmlHttpRequest': GM_xmlhttpRequest, + } + for (newKey in entries) { + let old = entries[newKey]; + if (old && (typeof GM[newKey] == 'undefined')) { + GM[newKey] = function(...args) { + return new Promise((resolve, reject) => { + try { + resolve(old(...args)); + } catch (e) { + reject(e); + } + }); + }; + } + }; + const unsafeWindow = window; // ====== The actual user script source ====== // diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index d30514f83..f9f02ba8b 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -507,3 +507,12 @@ class ModelValidator: @pytest.fixture def model_validator(qtmodeltester): return ModelValidator(qtmodeltester) + + +@pytest.fixture +def download_stub(win_registry, tmpdir, stubs): + """Register a FakeDownloadManager.""" + stub = stubs.FakeDownloadManager(tmpdir) + objreg.register('qtnetwork-download-manager', stub) + yield stub + objreg.delete('qtnetwork-download-manager') diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 3957a670a..fbe7035e3 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -22,6 +22,8 @@ """Fake objects/stubs.""" from unittest import mock +import contextlib +import shutil import attr from PyQt5.QtCore import pyqtSignal, QPoint, QProcess, QObject, QUrl @@ -29,7 +31,7 @@ from PyQt5.QtNetwork import (QNetworkRequest, QAbstractNetworkCache, QNetworkCacheMetaData) from PyQt5.QtWidgets import QCommonStyle, QLineEdit, QWidget, QTabBar -from qutebrowser.browser import browsertab +from qutebrowser.browser import browsertab, downloads from qutebrowser.utils import usertypes from qutebrowser.mainwindow import mainwindow @@ -558,3 +560,49 @@ class HTTPPostStub(QObject): def post(self, url, data=None): self.url = url self.data = data + + +class FakeDownloadItem(QObject): + + """Mock browser.downloads.DownloadItem.""" + + finished = pyqtSignal() + + def __init__(self, fileobj, name, parent=None): + super().__init__(parent) + self.fileobj = fileobj + self.name = name + self.successful = False + + +class FakeDownloadManager: + + """Mock browser.downloads.DownloadManager.""" + + def __init__(self, tmpdir): + self._tmpdir = tmpdir + self.downloads = [] + + @contextlib.contextmanager + def _open_fileobj(self, target): + """Ensure a DownloadTarget's fileobj attribute is available.""" + if isinstance(target, downloads.FileDownloadTarget): + target.fileobj = open(target.filename, 'wb') + try: + yield target.fileobj + finally: + target.fileobj.close() + else: + yield target.fileobj + + def get(self, url, target, **kwargs): + """Return a FakeDownloadItem instance with a fileobj. + + The content is copied from the file the given url links to. + """ + with self._open_fileobj(target): + download_item = FakeDownloadItem(target.fileobj, name=url.path()) + with (self._tmpdir / url.path()).open('rb') as fake_url_file: + shutil.copyfileobj(fake_url_file, download_item.fileobj) + self.downloads.append(download_item) + return download_item diff --git a/tests/unit/browser/test_adblock.py b/tests/unit/browser/test_adblock.py index 09161e806..5b353efb9 100644 --- a/tests/unit/browser/test_adblock.py +++ b/tests/unit/browser/test_adblock.py @@ -21,15 +21,13 @@ import os import os.path import zipfile -import shutil import logging import pytest -from PyQt5.QtCore import pyqtSignal, QUrl, QObject +from PyQt5.QtCore import QUrl from qutebrowser.browser import adblock -from qutebrowser.utils import objreg pytestmark = pytest.mark.usefixtures('qapp', 'config_tmpdir') @@ -69,46 +67,6 @@ def basedir(fake_args): fake_args.basedir = None -class FakeDownloadItem(QObject): - - """Mock browser.downloads.DownloadItem.""" - - finished = pyqtSignal() - - def __init__(self, fileobj, name, parent=None): - super().__init__(parent) - self.fileobj = fileobj - self.name = name - self.successful = True - - -class FakeDownloadManager: - - """Mock browser.downloads.DownloadManager.""" - - def __init__(self, tmpdir): - self._tmpdir = tmpdir - - def get(self, url, target, **kwargs): - """Return a FakeDownloadItem instance with a fileobj. - - The content is copied from the file the given url links to. - """ - download_item = FakeDownloadItem(target.fileobj, name=url.path()) - with (self._tmpdir / url.path()).open('rb') as fake_url_file: - shutil.copyfileobj(fake_url_file, download_item.fileobj) - return download_item - - -@pytest.fixture -def download_stub(win_registry, tmpdir): - """Register a FakeDownloadManager.""" - stub = FakeDownloadManager(tmpdir) - objreg.register('qtnetwork-download-manager', stub) - yield - objreg.delete('qtnetwork-download-manager') - - def create_zipfile(directory, files, zipname='test'): """Return a path to a newly created zip file. @@ -248,6 +206,7 @@ def test_disabled_blocking_update(basedir, config_stub, download_stub, while host_blocker._in_progress: current_download = host_blocker._in_progress[0] with caplog.at_level(logging.ERROR): + current_download.successful = True current_download.finished.emit() host_blocker.read_hosts() for str_url in URLS_TO_CHECK: @@ -263,6 +222,8 @@ def test_no_blocklist_update(config_stub, download_stub, host_blocker = adblock.HostBlocker() host_blocker.adblock_update() host_blocker.read_hosts() + for dl in download_stub.downloads: + dl.successful = True for str_url in URLS_TO_CHECK: assert not host_blocker.is_blocked(QUrl(str_url)) @@ -280,6 +241,7 @@ def test_successful_update(config_stub, basedir, download_stub, while host_blocker._in_progress: current_download = host_blocker._in_progress[0] with caplog.at_level(logging.ERROR): + current_download.successful = True current_download.finished.emit() host_blocker.read_hosts() assert_urls(host_blocker, whitelisted=[]) @@ -307,6 +269,8 @@ def test_failed_dl_update(config_stub, basedir, download_stub, # if current download is the file we want to fail, make it fail if current_download.name == dl_fail_blocklist.path(): current_download.successful = False + else: + current_download.successful = True with caplog.at_level(logging.ERROR): current_download.finished.emit() host_blocker.read_hosts() @@ -336,16 +300,18 @@ def test_invalid_utf8(config_stub, download_stub, tmpdir, data_tmpdir, host_blocker = adblock.HostBlocker() host_blocker.adblock_update() - finished_signal = host_blocker._in_progress[0].finished + current_download = host_blocker._in_progress[0] if location == 'content': with caplog.at_level(logging.ERROR): - finished_signal.emit() + current_download.successful = True + current_download.finished.emit() expected = (r"Failed to decode: " r"b'https://www.example.org/\xa0localhost") assert caplog.records[-2].message.startswith(expected) else: - finished_signal.emit() + current_download.successful = True + current_download.finished.emit() host_blocker.read_hosts() assert_urls(host_blocker, whitelisted=[]) diff --git a/tests/unit/javascript/test_greasemonkey.py b/tests/unit/javascript/test_greasemonkey.py index 52af51a4b..7759f5d18 100644 --- a/tests/unit/javascript/test_greasemonkey.py +++ b/tests/unit/javascript/test_greasemonkey.py @@ -128,3 +128,33 @@ def test_load_emits_signal(qtbot): gm_manager = greasemonkey.GreasemonkeyManager() with qtbot.wait_signal(gm_manager.scripts_reloaded): gm_manager.load_scripts() + + +def test_required_scripts_are_included(download_stub, tmpdir): + test_require_script = textwrap.dedent(""" + // ==UserScript== + // @name qutebrowser test userscript + // @namespace invalid.org + // @include http://localhost:*/data/title.html + // @match http://trolol* + // @exclude https://badhost.xxx/* + // @run-at document-start + // @require http://localhost/test.js + // ==/UserScript== + console.log("Script is running."); + """) + _save_script(test_require_script, 'requiring.user.js') + with open(str(tmpdir / 'test.js'), 'w', encoding='UTF-8') as f: + f.write("REQUIRED SCRIPT") + + gm_manager = greasemonkey.GreasemonkeyManager() + assert len(gm_manager._in_progress_dls) == 1 + for download in gm_manager._in_progress_dls: + download.finished.emit() + + scripts = gm_manager.all_scripts() + assert len(scripts) == 1 + assert "REQUIRED SCRIPT" in scripts[0].code() + # Additionally check that the base script is still being parsed correctly + assert "Script is running." in scripts[0].code() + assert scripts[0].excludes