rnhmjoj/qutebrowser
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'origin/pr/3456'

Browse Source
This commit is contained in:
Florian Bruhin 2018-03-06 09:32:39 +01:00
commit 8c0bca90d3
7 changed files with 277 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
qutebrowser/browser/downloads.py
View File

@ -238,11 +238,14 @@ class FileDownloadTarget(_DownloadTarget):
Attributes:
filename: Filename where the download should be saved.
force_overwrite: Whether to overwrite the target without
prompting the user.
"""
def __init__(self, filename):
def __init__(self, filename, force_overwrite=False):
# pylint: disable=super-init-not-called
self.filename = filename
self.force_overwrite = force_overwrite
def suggested_filename(self):
return os.path.basename(self.filename)
@ -738,7 +741,8 @@ class AbstractDownloadItem(QObject):
if isinstance(target, FileObjDownloadTarget):
self._set_fileobj(target.fileobj, autoclose=False)
elif isinstance(target, FileDownloadTarget):
self._set_filename(target.filename)
self._set_filename(
target.filename, force_overwrite=target.force_overwrite)
elif isinstance(target, OpenFileDownloadTarget):
try:
fobj = temp_download_manager.get_tmpfile(self.basename)

152
qutebrowser/browser/greasemonkey.py
View File

@ -23,13 +23,16 @@ import re
import os
import json
import fnmatch
import functools
import glob
import textwrap
import attr
from PyQt5.QtCore import pyqtSignal, QObject, QUrl
from qutebrowser.utils import log, standarddir, jinja, objreg
from qutebrowser.utils import log, standarddir, jinja, objreg, utils
from qutebrowser.commands import cmdutils
from qutebrowser.browser import downloads
def _scripts_dir():
@ -45,6 +48,7 @@ class GreasemonkeyScript:
self._code = code
self.includes = []
self.excludes = []
self.requires = []
self.description = None
self.name = None
self.namespace = None
@ -66,6 +70,8 @@ class GreasemonkeyScript:
self.run_at = value
elif name == 'noframes':
self.runs_on_sub_frames = False
elif name == 'require':
self.requires.append(value)
HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n'
PROPS_REGEX = r'// @(?P<prop>[^\s]+)\s*(?P<val>.*)'
@ -93,7 +99,7 @@ class GreasemonkeyScript:
"""Return the processed JavaScript code of this script.
Adorns the source code with GM_* methods for Greasemonkey
compatibility and wraps it in an IFFE to hide it within a
compatibility and wraps it in an IIFE to hide it within a
lexical scope. Note that this means line numbers in your
browser's debugger/inspector will not match up to the line
numbers in the source script directly.
@ -115,6 +121,14 @@ class GreasemonkeyScript:
'run-at': self.run_at,
})
def add_required_script(self, source):
"""Add the source of a required script to this script."""
# The additional source is indented in case it also contains a
# metadata block. Because we pass everything at once to
# QWebEngineScript and that would parse the first metadata block
# found as the valid one.
self._code = "\n".join([textwrap.indent(source, " "), self._code])
@attr.s
class MatchingScripts(object):
@ -145,15 +159,24 @@ class GreasemonkeyManager(QObject):
def __init__(self, parent=None):
super().__init__(parent)
self._run_start = []
self._run_end = []
self._run_idle = []
self._in_progress_dls = []
self.load_scripts()
@cmdutils.register(name='greasemonkey-reload',
instance='greasemonkey')
def load_scripts(self):
def load_scripts(self, force=False):
"""Re-read Greasemonkey scripts from disk.
The scripts are read from a 'greasemonkey' subdirectory in
qutebrowser's data directory (see `:version`).
Args:
force: For any scripts that have required dependencies,
re-download them.
"""
self._run_start = []
self._run_end = []
@ -169,24 +192,115 @@ class GreasemonkeyManager(QObject):
script = GreasemonkeyScript.parse(script_file.read())
if not script.name:
script.name = script_filename
if script.run_at == 'document-start':
self._run_start.append(script)
elif script.run_at == 'document-end':
self._run_end.append(script)
elif script.run_at == 'document-idle':
self._run_idle.append(script)
else:
if script.run_at:
log.greasemonkey.warning(
"Script {} has invalid run-at defined, "
"defaulting to document-end".format(script_path))
# Default as per
# https://wiki.greasespot.net/Metadata_Block#.40run-at
self._run_end.append(script)
log.greasemonkey.debug("Loaded script: {}".format(script.name))
self.add_script(script, force)
self.scripts_reloaded.emit()
def add_script(self, script, force=False):
"""Add a GreasemonkeyScript to this manager.
Args:
force: Fetch and overwrite any dependancies which are
already locally cached.
"""
if script.requires:
log.greasemonkey.debug(
"Deferring script until requirements are "
"fulfilled: {}".format(script.name))
self._get_required_scripts(script, force)
else:
self._add_script(script)
def _add_script(self, script):
if script.run_at == 'document-start':
self._run_start.append(script)
elif script.run_at == 'document-end':
self._run_end.append(script)
elif script.run_at == 'document-idle':
self._run_idle.append(script)
else:
if script.run_at:
log.greasemonkey.warning("Script {} has invalid run-at "
"defined, defaulting to "
"document-end"
.format(script.name))
# Default as per
# https://wiki.greasespot.net/Metadata_Block#.40run-at
self._run_end.append(script)
log.greasemonkey.debug("Loaded script: {}".format(script.name))
def _required_url_to_file_path(self, url):
requires_dir = os.path.join(_scripts_dir(), 'requires')
if not os.path.exists(requires_dir):
os.mkdir(requires_dir)
return os.path.join(requires_dir, utils.sanitize_filename(url))
def _on_required_download_finished(self, script, download):
self._in_progress_dls.remove(download)
if not self._add_script_with_requires(script):
log.greasemonkey.debug(
"Finished download {} for script {} "
"but some requirements are still pending"
.format(download.basename, script.name))
def _add_script_with_requires(self, script, quiet=False):
"""Add a script with pending downloads to this GreasemonkeyManager.
Specifically a script that has dependancies specified via an
`@require` rule.
Args:
script: The GreasemonkeyScript to add.
quiet: True to suppress the scripts_reloaded signal after
adding `script`.
Returns: True if the script was added, False if there are still
dependancies being downloaded.
"""
# See if we are still waiting on any required scripts for this one
for dl in self._in_progress_dls:
if dl.requested_url in script.requires:
return False
# Need to add the required scripts to the IIFE now
for url in reversed(script.requires):
target_path = self._required_url_to_file_path(url)
log.greasemonkey.debug(
"Adding required script for {} to IIFE: {}"
.format(script.name, url))
with open(target_path, encoding='utf8') as f:
script.add_required_script(f.read())
self._add_script(script)
if not quiet:
self.scripts_reloaded.emit()
return True
def _get_required_scripts(self, script, force=False):
required_dls = [(url, self._required_url_to_file_path(url))
for url in script.requires]
if not force:
required_dls = [(url, path) for (url, path) in required_dls
if not os.path.exists(path)]
if not required_dls:
# All the required files exist already
self._add_script_with_requires(script, quiet=True)
return
download_manager = objreg.get('qtnetwork-download-manager')
for url, target_path in required_dls:
target = downloads.FileDownloadTarget(target_path,
force_overwrite=True)
download = download_manager.get(QUrl(url), target=target,
auto_remove=True)
download.requested_url = url
self._in_progress_dls.append(download)
if download.successful:
self._on_required_download_finished(script, download)
else:
download.finished.connect(
functools.partial(self._on_required_download_finished,
script, download))
def scripts_for(self, url):
"""Fetch scripts that are registered to run for url.

38
qutebrowser/javascript/greasemonkey_wrapper.js
View File

@ -110,6 +110,44 @@
}
}
// Stub these two so that the gm4 polyfill script doesn't try to
// create broken versions as attributes of window.
function GM_getResourceText(caption, commandFunc, accessKey) {
console.error(`${GM_info.script.name} called unimplemented GM_getResourceText`);
}
function GM_registerMenuCommand(caption, commandFunc, accessKey) {
console.error(`${GM_info.script.name} called unimplemented GM_registerMenuCommand`);
}
// Mock the greasemonkey 4.0 async API.
const GM = {};
GM.info = GM_info;
const entries = {
'log': GM_log,
'addStyle': GM_addStyle,
'deleteValue': GM_deleteValue,
'getValue': GM_getValue,
'listValues': GM_listValues,
'openInTab': GM_openInTab,
'setValue': GM_setValue,
'xmlHttpRequest': GM_xmlhttpRequest,
}
for (newKey in entries) {
let old = entries[newKey];
if (old && (typeof GM[newKey] == 'undefined')) {
GM[newKey] = function(...args) {
return new Promise((resolve, reject) => {
try {
resolve(old(...args));
} catch (e) {
reject(e);
}
});
};
}
};
const unsafeWindow = window;
// ====== The actual user script source ====== //

9
tests/helpers/fixtures.py
View File

@ -507,3 +507,12 @@ class ModelValidator:
@pytest.fixture
def model_validator(qtmodeltester):
return ModelValidator(qtmodeltester)
@pytest.fixture
def download_stub(win_registry, tmpdir, stubs):
"""Register a FakeDownloadManager."""
stub = stubs.FakeDownloadManager(tmpdir)
objreg.register('qtnetwork-download-manager', stub)
yield stub
objreg.delete('qtnetwork-download-manager')

50
tests/helpers/stubs.py
View File

@ -22,6 +22,8 @@
"""Fake objects/stubs."""
from unittest import mock
import contextlib
import shutil
import attr
from PyQt5.QtCore import pyqtSignal, QPoint, QProcess, QObject, QUrl
@ -29,7 +31,7 @@ from PyQt5.QtNetwork import (QNetworkRequest, QAbstractNetworkCache,
QNetworkCacheMetaData)
from PyQt5.QtWidgets import QCommonStyle, QLineEdit, QWidget, QTabBar
from qutebrowser.browser import browsertab
from qutebrowser.browser import browsertab, downloads
from qutebrowser.utils import usertypes
from qutebrowser.mainwindow import mainwindow
@ -558,3 +560,49 @@ class HTTPPostStub(QObject):
def post(self, url, data=None):
self.url = url
self.data = data
class FakeDownloadItem(QObject):
"""Mock browser.downloads.DownloadItem."""
finished = pyqtSignal()
def __init__(self, fileobj, name, parent=None):
super().__init__(parent)
self.fileobj = fileobj
self.name = name
self.successful = False
class FakeDownloadManager:
"""Mock browser.downloads.DownloadManager."""
def __init__(self, tmpdir):
self._tmpdir = tmpdir
self.downloads = []
@contextlib.contextmanager
def _open_fileobj(self, target):
"""Ensure a DownloadTarget's fileobj attribute is available."""
if isinstance(target, downloads.FileDownloadTarget):
target.fileobj = open(target.filename, 'wb')
try:
yield target.fileobj
finally:
target.fileobj.close()
else:
yield target.fileobj
def get(self, url, target, **kwargs):
"""Return a FakeDownloadItem instance with a fileobj.
The content is copied from the file the given url links to.
"""
with self._open_fileobj(target):
download_item = FakeDownloadItem(target.fileobj, name=url.path())
with (self._tmpdir / url.path()).open('rb') as fake_url_file:
shutil.copyfileobj(fake_url_file, download_item.fileobj)
self.downloads.append(download_item)
return download_item

58
tests/unit/browser/test_adblock.py
View File

@ -21,15 +21,13 @@
import os
import os.path
import zipfile
import shutil
import logging
import pytest
from PyQt5.QtCore import pyqtSignal, QUrl, QObject
from PyQt5.QtCore import QUrl
from qutebrowser.browser import adblock
from qutebrowser.utils import objreg
pytestmark = pytest.mark.usefixtures('qapp', 'config_tmpdir')
@ -69,46 +67,6 @@ def basedir(fake_args):
fake_args.basedir = None
class FakeDownloadItem(QObject):
"""Mock browser.downloads.DownloadItem."""
finished = pyqtSignal()
def __init__(self, fileobj, name, parent=None):
super().__init__(parent)
self.fileobj = fileobj
self.name = name
self.successful = True
class FakeDownloadManager:
"""Mock browser.downloads.DownloadManager."""
def __init__(self, tmpdir):
self._tmpdir = tmpdir
def get(self, url, target, **kwargs):
"""Return a FakeDownloadItem instance with a fileobj.
The content is copied from the file the given url links to.
"""
download_item = FakeDownloadItem(target.fileobj, name=url.path())
with (self._tmpdir / url.path()).open('rb') as fake_url_file:
shutil.copyfileobj(fake_url_file, download_item.fileobj)
return download_item
@pytest.fixture
def download_stub(win_registry, tmpdir):
"""Register a FakeDownloadManager."""
stub = FakeDownloadManager(tmpdir)
objreg.register('qtnetwork-download-manager', stub)
yield
objreg.delete('qtnetwork-download-manager')
def create_zipfile(directory, files, zipname='test'):
"""Return a path to a newly created zip file.
@ -248,6 +206,7 @@ def test_disabled_blocking_update(basedir, config_stub, download_stub,
while host_blocker._in_progress:
current_download = host_blocker._in_progress[0]
with caplog.at_level(logging.ERROR):
current_download.successful = True
current_download.finished.emit()
host_blocker.read_hosts()
for str_url in URLS_TO_CHECK:
@ -263,6 +222,8 @@ def test_no_blocklist_update(config_stub, download_stub,
host_blocker = adblock.HostBlocker()
host_blocker.adblock_update()
host_blocker.read_hosts()
for dl in download_stub.downloads:
dl.successful = True
for str_url in URLS_TO_CHECK:
assert not host_blocker.is_blocked(QUrl(str_url))
@ -280,6 +241,7 @@ def test_successful_update(config_stub, basedir, download_stub,
while host_blocker._in_progress:
current_download = host_blocker._in_progress[0]
with caplog.at_level(logging.ERROR):
current_download.successful = True
current_download.finished.emit()
host_blocker.read_hosts()
assert_urls(host_blocker, whitelisted=[])
@ -307,6 +269,8 @@ def test_failed_dl_update(config_stub, basedir, download_stub,
# if current download is the file we want to fail, make it fail
if current_download.name == dl_fail_blocklist.path():
current_download.successful = False
else:
current_download.successful = True
with caplog.at_level(logging.ERROR):
current_download.finished.emit()
host_blocker.read_hosts()
@ -336,16 +300,18 @@ def test_invalid_utf8(config_stub, download_stub, tmpdir, data_tmpdir,
host_blocker = adblock.HostBlocker()
host_blocker.adblock_update()
finished_signal = host_blocker._in_progress[0].finished
current_download = host_blocker._in_progress[0]
if location == 'content':
with caplog.at_level(logging.ERROR):
finished_signal.emit()
current_download.successful = True
current_download.finished.emit()
expected = (r"Failed to decode: "
r"b'https://www.example.org/\xa0localhost")
assert caplog.records[-2].message.startswith(expected)
else:
finished_signal.emit()
current_download.successful = True
current_download.finished.emit()
host_blocker.read_hosts()
assert_urls(host_blocker, whitelisted=[])

30
tests/unit/javascript/test_greasemonkey.py
View File

@ -128,3 +128,33 @@ def test_load_emits_signal(qtbot):
gm_manager = greasemonkey.GreasemonkeyManager()
with qtbot.wait_signal(gm_manager.scripts_reloaded):
gm_manager.load_scripts()
def test_required_scripts_are_included(download_stub, tmpdir):
test_require_script = textwrap.dedent("""
// ==UserScript==
// @name qutebrowser test userscript
// @namespace invalid.org
// @include http://localhost:*/data/title.html
// @match http://trolol*
// @exclude https://badhost.xxx/*
// @run-at document-start
// @require http://localhost/test.js
// ==/UserScript==
console.log("Script is running.");
""")
_save_script(test_require_script, 'requiring.user.js')
with open(str(tmpdir / 'test.js'), 'w', encoding='UTF-8') as f:
f.write("REQUIRED SCRIPT")
gm_manager = greasemonkey.GreasemonkeyManager()
assert len(gm_manager._in_progress_dls) == 1
for download in gm_manager._in_progress_dls:
download.finished.emit()
scripts = gm_manager.all_scripts()
assert len(scripts) == 1
assert "REQUIRED SCRIPT" in scripts[0].code()
# Additionally check that the base script is still being parsed correctly
assert "Script is running." in scripts[0].code()
assert scripts[0].excludes