From 8cb6b832d1e1311ee4404a44ad2f471eaa00c884 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 00:24:59 -0400 Subject: [PATCH 001/415] script to import history data from other browsers --- scripts/hist_importer.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 scripts/hist_importer.py diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py new file mode 100644 index 000000000..e69de29bb From c6d140a40a77345c484bc8557f84f97d33539bfe Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 00:26:47 -0400 Subject: [PATCH 002/415] adding script to import history data from other browsers --- scripts/hist_importer.py | 135 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index e69de29bb..5f4ead361 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# This file is part of qutebrowser. + +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + + +''' +Tool to import browser history data from other browsers. Although, safari +support is still on the way. +''' + + +import argparse +import sqlite3 +import sys + + +def parser(): + """Parse command line arguments.""" + description = 'This program is meant to extract browser history from your'\ + 'previous browser and import them into qutebrowser.' + epilog = 'Databases:\n\tQute: Is named "history.sqlite" and can be found '\ + 'at your --basedir. In order to find where your basedir is you '\ + 'can run ":open qute:version" inside qutebrowser.'\ + '\n\tFirerox: Is named "places.sqlite", and can be found at your'\ + 'system\'s profile folder. Check this link for where it is locat'\ + 'ed: http://kb.mozillazine.org/Profile_folder'\ + '\n\tChrome: Is named "History", and can be found at the respec'\ + 'tive User Data Directory. Check this link for where it is locat'\ + 'ed: https://chromium.googlesource.com/chromium/src/+/master/'\ + 'docs/user_data_dir.md\n\n'\ + 'Example: $this_script.py -b firefox -s /Firefox/Profile/places.'\ + 'sqlite -d /qutebrowser/data/history.sqlite' + parser = argparse.ArgumentParser( + description=description, epilog=epilog, + formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument('-b', '--browser', dest='browser', required=True, + type=str, help='Browsers: {firefox, chrome, safari}') + parser.add_argument('-s', '--source', dest='source', required=True, + type=str, help='Source: fullpath to the sqlite data' + 'base file from the source browser.') + parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, + help='Destination: The fullpath to the qutebrowser ' + 'sqlite database') + return parser.parse_args() + + +def open_db(db): + """Open connection with database.""" + try: + conn = sqlite3.connect(db) + return conn + except Exception as e: + print('Error: {}'.format(e)) + raise('Error: There was some error trying to to connect with the [{}]' + 'database. Verify if the filepath is correct or is being used.'. + format(db)) + + +def extract(source, query): + """Performs extraction of (datetime,url,title) from source.""" + try: + conn = open_db(source) + cursor = conn.cursor() + cursor.execute(query) + history = cursor.fetchall() + conn.close() + return history + except Exception as e: + # print('Error: {}'.format(e)) + print(type(source)) + raise('Error: There was some error trying to to connect with the [{}]' + 'database. Verify if the filepath is correct or is being used.'. + format(str(source))) + + +def clean(history): + """Receives a list of records:(datetime,url,title). And clean all records + in place, that has a NULL/None datetime attribute. Otherwise Qutebrowser + will throw errors.""" + nulls = [record for record in history if record[0] is None] + for null_datetime in nulls: + history.remove(null_datetime) + return history + + +def insert_qb(history, dest): + conn = open_db(dest) + cursor = conn.cursor() + cursor.executemany( + 'INSERT INTO History (url,title,atime) VALUES (?,?,?)', history + ) + conn.commit() + conn.close() + + +def main(): + args = parser() + browser = args.browser.lower() + source, dest = args.source, args.dest + query = { + 'firefox': 'select url,title,last_visit_date/1000000 as date ' + 'from moz_places', + 'chrome': 'select url,title,last_visit_time/10000000 as date ' + 'from urls', + 'safari': None + } + if browser not in query: + sys.exit('Sorry, the selected browser: "{}" is not supported.'.format( + browser)) + else: + if browser == 'safari': + print('Sorry, currently we do not support this browser.') + sys.exit(1) + history = extract(source, query[browser]) + history = clean(history) + insert_qb(history, dest) + + +if __name__ == "__main__": + main() From 4dc232f259d4a24da075e584998998873e58c221 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 13:54:24 -0400 Subject: [PATCH 003/415] pylint fixes --- scripts/hist_importer.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 5f4ead361..44a76bd8a 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -17,10 +17,8 @@ # along with qutebrowser. If not, see . -''' -Tool to import browser history data from other browsers. Although, safari -support is still on the way. -''' +'''Tool to import browser history data from other browsers. Although, safari +support is still on the way.''' import argparse @@ -44,31 +42,31 @@ def parser(): 'docs/user_data_dir.md\n\n'\ 'Example: $this_script.py -b firefox -s /Firefox/Profile/places.'\ 'sqlite -d /qutebrowser/data/history.sqlite' - parser = argparse.ArgumentParser( + parsed = argparse.ArgumentParser( description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter ) - parser.add_argument('-b', '--browser', dest='browser', required=True, + parsed.add_argument('-b', '--browser', dest='browser', required=True, type=str, help='Browsers: {firefox, chrome, safari}') - parser.add_argument('-s', '--source', dest='source', required=True, + parsed.add_argument('-s', '--source', dest='source', required=True, type=str, help='Source: fullpath to the sqlite data' 'base file from the source browser.') - parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, + parsed.add_argument('-d', '--dest', dest='dest', required=True, type=str, help='Destination: The fullpath to the qutebrowser ' 'sqlite database') - return parser.parse_args() + return parsed.parse_args() -def open_db(db): +def open_db(data_base): """Open connection with database.""" try: - conn = sqlite3.connect(db) + conn = sqlite3.connect(data_base) return conn - except Exception as e: - print('Error: {}'.format(e)) + except Exception as any_e: + print('Error: {}'.format(any_e)) raise('Error: There was some error trying to to connect with the [{}]' 'database. Verify if the filepath is correct or is being used.'. - format(db)) + format(data_base)) def extract(source, query): @@ -80,8 +78,8 @@ def extract(source, query): history = cursor.fetchall() conn.close() return history - except Exception as e: - # print('Error: {}'.format(e)) + except Exception as any_e: + print('Error: {}'.format(any_e)) print(type(source)) raise('Error: There was some error trying to to connect with the [{}]' 'database. Verify if the filepath is correct or is being used.'. @@ -99,6 +97,8 @@ def clean(history): def insert_qb(history, dest): + """Given a list of records in history and a dest db, insert all records in + the dest db.""" conn = open_db(dest) cursor = conn.cursor() cursor.executemany( @@ -109,19 +109,20 @@ def insert_qb(history, dest): def main(): + """Main control flux of the script.""" args = parser() browser = args.browser.lower() source, dest = args.source, args.dest query = { 'firefox': 'select url,title,last_visit_date/1000000 as date ' - 'from moz_places', + 'from moz_places', 'chrome': 'select url,title,last_visit_time/10000000 as date ' - 'from urls', + 'from urls', 'safari': None } if browser not in query: sys.exit('Sorry, the selected browser: "{}" is not supported.'.format( - browser)) + browser)) else: if browser == 'safari': print('Sorry, currently we do not support this browser.') From 665a76561ecf405783f13beca694085a0803e479 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 22:50:52 -0400 Subject: [PATCH 004/415] add insertions to ComandHistory table as well --- scripts/hist_importer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 44a76bd8a..8df16384a 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -104,6 +104,10 @@ def insert_qb(history, dest): cursor.executemany( 'INSERT INTO History (url,title,atime) VALUES (?,?,?)', history ) + cursor.executemany( + 'INSERT INTO CompletionHistory (url,title,last_atime) VALUES (?,?,?)', + history + ) conn.commit() conn.close() From 92f9a8503ee90cba87bc11b9aa983e8c8069c497 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Tue, 3 Oct 2017 01:55:31 -0400 Subject: [PATCH 005/415] add required redirect (url,title,atime,redirect) --- scripts/hist_importer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 8df16384a..d96284879 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -89,10 +89,14 @@ def extract(source, query): def clean(history): """Receives a list of records:(datetime,url,title). And clean all records in place, that has a NULL/None datetime attribute. Otherwise Qutebrowser - will throw errors.""" + will throw errors. Also, will add a 4rth attribute of '0' for the redirect + field in history.sqlite in qutebrowser.""" nulls = [record for record in history if record[0] is None] for null_datetime in nulls: history.remove(null_datetime) + history = [list(record) for record in history] + for record in history: + record.append('0') return history @@ -102,12 +106,10 @@ def insert_qb(history, dest): conn = open_db(dest) cursor = conn.cursor() cursor.executemany( - 'INSERT INTO History (url,title,atime) VALUES (?,?,?)', history - ) - cursor.executemany( - 'INSERT INTO CompletionHistory (url,title,last_atime) VALUES (?,?,?)', + 'INSERT INTO History (url,title,atime,redirect) VALUES (?,?,?,?)', history ) + cursor.execute('DROP TABLE CompletionHistory') conn.commit() conn.close() From 51d48f6b00c5528b62a2c24ec7973b380c628965 Mon Sep 17 00:00:00 2001 From: Ulrik de Muelenaere Date: Sat, 28 Oct 2017 22:16:29 +0200 Subject: [PATCH 006/415] Rewrite user stylesheet injection for WebEngine This now works correctly in XML documents. The stylesheet is applied at document creation to reduce flickering, and is updated if the user_stylesheets setting is changed after page load. --- .../browser/webengine/webenginesettings.py | 33 ++-- qutebrowser/browser/webengine/webenginetab.py | 2 +- qutebrowser/javascript/stylesheet.js | 142 ++++++++++++++++++ .../mhtml/simple/simple-webengine.mht | 6 +- 4 files changed, 167 insertions(+), 16 deletions(-) create mode 100644 qutebrowser/javascript/stylesheet.js diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 12503a7c0..d48c58cf4 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -37,7 +37,7 @@ from qutebrowser.browser import shared from qutebrowser.browser.webengine import spell from qutebrowser.config import config, websettings from qutebrowser.utils import (utils, standarddir, javascript, qtutils, - message, log) + message, log, objreg) # The default QWebEngineProfile default_profile = None @@ -153,33 +153,41 @@ class DictionaryLanguageSetter(DefaultProfileSetter): def _init_stylesheet(profile): """Initialize custom stylesheets. - Mostly inspired by QupZilla: + Partially inspired by QupZilla: https://github.com/QupZilla/qupzilla/blob/v2.0/src/lib/app/mainapplication.cpp#L1063-L1101 - https://github.com/QupZilla/qupzilla/blob/v2.0/src/lib/tools/scripts.cpp#L119-L132 """ old_script = profile.scripts().findScript('_qute_stylesheet') if not old_script.isNull(): profile.scripts().remove(old_script) css = shared.get_user_stylesheet() - source = """ - (function() {{ - var css = document.createElement('style'); - css.setAttribute('type', 'text/css'); - css.appendChild(document.createTextNode('{}')); - document.getElementsByTagName('head')[0].appendChild(css); - }})() - """.format(javascript.string_escape(css)) + source = '\n'.join([ + '"use strict";', + 'window._qutebrowser = window._qutebrowser || {};', + utils.read_file('javascript/stylesheet.js'), + javascript.assemble('stylesheet', 'set_css', css), + ]) script = QWebEngineScript() script.setName('_qute_stylesheet') - script.setInjectionPoint(QWebEngineScript.DocumentReady) + script.setInjectionPoint(QWebEngineScript.DocumentCreation) script.setWorldId(QWebEngineScript.ApplicationWorld) script.setRunsOnSubFrames(True) script.setSourceCode(source) profile.scripts().insert(script) +def _update_stylesheet(): + """Update the custom stylesheet in existing tabs.""" + css = shared.get_user_stylesheet() + code = javascript.assemble('stylesheet', 'set_css', css) + for win_id in objreg.window_registry: + tab_registry = objreg.get('tab-registry', scope='window', + window=win_id) + for tab in tab_registry.values(): + tab.run_js_async(code) + + def _set_http_headers(profile): """Set the user agent and accept-language for the given profile. @@ -199,6 +207,7 @@ def _update_settings(option): if option in ['scrolling.bar', 'content.user_stylesheets']: _init_stylesheet(default_profile) _init_stylesheet(private_profile) + _update_stylesheet() elif option in ['content.headers.user_agent', 'content.headers.accept_language']: _set_http_headers(default_profile) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 15466ab0d..22e17e2e7 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -533,7 +533,7 @@ class WebEngineTab(browsertab.AbstractTab): def _init_js(self): js_code = '\n'.join([ '"use strict";', - 'window._qutebrowser = {};', + 'window._qutebrowser = window._qutebrowser || {};', utils.read_file('javascript/scroll.js'), utils.read_file('javascript/webelem.js'), ]) diff --git a/qutebrowser/javascript/stylesheet.js b/qutebrowser/javascript/stylesheet.js new file mode 100644 index 000000000..3f2bc78a0 --- /dev/null +++ b/qutebrowser/javascript/stylesheet.js @@ -0,0 +1,142 @@ +/** + * Copyright 2017 Ulrik de Muelenaere + * + * This file is part of qutebrowser. + * + * qutebrowser is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * qutebrowser is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with qutebrowser. If not, see . + */ + +"use strict"; + +window._qutebrowser.stylesheet = (function() { + if (window._qutebrowser.stylesheet) { + return window._qutebrowser.stylesheet; + } + + var funcs = {}; + + var xhtml_ns = "http://www.w3.org/1999/xhtml"; + var svg_ns = "http://www.w3.org/2000/svg"; + + var root_elem; + var style_elem; + var css_content = ""; + + var root_observer; + var style_observer; + var initialized = false; + + // Watch for rewrites of the root element and changes to its children, + // then move the stylesheet to the end. Partially inspired by Stylus: + // https://github.com/openstyles/stylus/blob/1.1.4.2/content/apply.js#L235-L355 + function watch_root() { + if (root_elem !== document.documentElement) { + root_elem = document.documentElement; + root_observer.disconnect(); + root_observer.observe(document, {"childList": true}); + root_observer.observe(root_elem, {"childList": true}); + } + if (style_elem !== root_elem.lastChild) { + root_elem.appendChild(style_elem); + } + } + + function create_style() { + var ns = xhtml_ns; + if (document.documentElement.namespaceURI === svg_ns) { + ns = svg_ns; + } + style_elem = document.createElementNS(ns, "style"); + style_elem.textContent = css_content; + root_observer = new MutationObserver(watch_root); + watch_root(); + } + + // We should only inject the stylesheet if the document already has style + // information associated with it. Otherwise we wait until the browser + // rewrites it to an XHTML document showing the document tree. As a + // starting point for exploring the relevant code in Chromium, see + // https://github.com/qt/qtwebengine-chromium/blob/cfe8c60/chromium/third_party/WebKit/Source/core/xml/parser/XMLDocumentParser.cpp#L1539-L1540 + function check_style(node) { + var stylesheet = node.nodeType === Node.PROCESSING_INSTRUCTION_NODE && + node.target === "xml-stylesheet" && + node.parentNode === document; + var known_ns = node.nodeType === Node.ELEMENT_NODE && + (node.namespaceURI === xhtml_ns || + node.namespaceURI === svg_ns); + if (stylesheet || known_ns) { + create_style(); + return true; + } + return false; + } + + function check_added_style(mutations) { + for (var mi = 0; mi < mutations.length; ++mi) { + var nodes = mutations[mi].addedNodes; + for (var ni = 0; ni < nodes.length; ++ni) { + if (check_style(nodes[ni])) { + style_observer.disconnect(); + return; + } + } + } + } + + function init() { + initialized = true; + // Chromium will not rewrite a document inside a frame, so add the + // stylesheet even if the document is unstyled. + if (window !== window.top) { + create_style(); + return; + } + var iter = document.createNodeIterator(document); + var node; + while ((node = iter.nextNode())) { + if (check_style(node)) { + return; + } + } + style_observer = new MutationObserver(check_added_style); + style_observer.observe(document, {"childList": true, "subtree": true}); + } + + var doc = document; + + funcs.set_css = function(css) { + if (!initialized) { + init(); + } + if (style_elem) { + style_elem.textContent = css; + // The browser seems to rewrite the document in same-origin frames + // without notifying the mutation observer. Ensure that the + // stylesheet is in the current document. + watch_root(); + } else { + css_content = css; + } + // Propagate the new CSS to all child frames. + // FIXME:qtwebengine This does not work for cross-origin frames. + for (var i = 0; i < window.frames.length; ++i) { + var frame = window.frames[i]; + if (frame._qutebrowser && frame._qutebrowser.stylesheet) { + frame._qutebrowser.stylesheet.set_css(css); + } + } + }; + + return funcs; +})(); diff --git a/tests/end2end/data/downloads/mhtml/simple/simple-webengine.mht b/tests/end2end/data/downloads/mhtml/simple/simple-webengine.mht index d8bfdee70..79bd1ae50 100644 --- a/tests/end2end/data/downloads/mhtml/simple/simple-webengine.mht +++ b/tests/end2end/data/downloads/mhtml/simple/simple-webengine.mht @@ -16,11 +16,11 @@ Content-Location: http://localhost:(port)/data/downloads/mhtml/simple/simple.htm t/html; charset=3DUTF-8"> =20 Simple MHTML test - + normal link to another page =20 - + -----=_qute-UUID From 2fe1a1db89f97c23b3ca1aeac8fb33f4d8c5f940 Mon Sep 17 00:00:00 2001 From: Ulrik de Muelenaere Date: Sun, 29 Oct 2017 00:23:11 +0200 Subject: [PATCH 007/415] Remove unused variable --- qutebrowser/javascript/stylesheet.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/qutebrowser/javascript/stylesheet.js b/qutebrowser/javascript/stylesheet.js index 3f2bc78a0..095a7f6bb 100644 --- a/qutebrowser/javascript/stylesheet.js +++ b/qutebrowser/javascript/stylesheet.js @@ -113,8 +113,6 @@ window._qutebrowser.stylesheet = (function() { style_observer.observe(document, {"childList": true, "subtree": true}); } - var doc = document; - funcs.set_css = function(css) { if (!initialized) { init(); From 0540a43995d5115af22b80b70e1689d56a054ca0 Mon Sep 17 00:00:00 2001 From: Ulrik de Muelenaere Date: Mon, 30 Oct 2017 19:52:15 +0200 Subject: [PATCH 008/415] Check for deleted window --- qutebrowser/browser/webengine/webenginesettings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index d48c58cf4..ff0beb805 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -29,6 +29,7 @@ Module attributes: import os +import sip from PyQt5.QtGui import QFont from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile, QWebEngineScript) @@ -181,7 +182,10 @@ def _update_stylesheet(): """Update the custom stylesheet in existing tabs.""" css = shared.get_user_stylesheet() code = javascript.assemble('stylesheet', 'set_css', css) - for win_id in objreg.window_registry: + for win_id, window in objreg.window_registry.items(): + # We could be in the middle of destroying a window here + if sip.isdeleted(window): + continue tab_registry = objreg.get('tab-registry', scope='window', window=win_id) for tab in tab_registry.values(): From 34b27437d0e18c5ec832a78dfac09291187f4fa7 Mon Sep 17 00:00:00 2001 From: Ulrik de Muelenaere Date: Mon, 30 Oct 2017 19:55:37 +0200 Subject: [PATCH 009/415] Clarify function names in stylesheet.js --- qutebrowser/javascript/stylesheet.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/qutebrowser/javascript/stylesheet.js b/qutebrowser/javascript/stylesheet.js index 095a7f6bb..e363d296f 100644 --- a/qutebrowser/javascript/stylesheet.js +++ b/qutebrowser/javascript/stylesheet.js @@ -75,18 +75,15 @@ window._qutebrowser.stylesheet = (function() { var known_ns = node.nodeType === Node.ELEMENT_NODE && (node.namespaceURI === xhtml_ns || node.namespaceURI === svg_ns); - if (stylesheet || known_ns) { - create_style(); - return true; - } - return false; + return stylesheet || known_ns; } - function check_added_style(mutations) { + function watch_added_style(mutations) { for (var mi = 0; mi < mutations.length; ++mi) { var nodes = mutations[mi].addedNodes; for (var ni = 0; ni < nodes.length; ++ni) { if (check_style(nodes[ni])) { + create_style(); style_observer.disconnect(); return; } @@ -106,10 +103,11 @@ window._qutebrowser.stylesheet = (function() { var node; while ((node = iter.nextNode())) { if (check_style(node)) { + create_style(); return; } } - style_observer = new MutationObserver(check_added_style); + style_observer = new MutationObserver(watch_added_style); style_observer.observe(document, {"childList": true, "subtree": true}); } From 3adc2e0f83bbba3c7b6db0619f2ee2ae90703db2 Mon Sep 17 00:00:00 2001 From: Ulrik de Muelenaere Date: Mon, 30 Oct 2017 19:56:12 +0200 Subject: [PATCH 010/415] Add filter to NodeIterator checking for styled nodes --- qutebrowser/javascript/stylesheet.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qutebrowser/javascript/stylesheet.js b/qutebrowser/javascript/stylesheet.js index e363d296f..384138f2f 100644 --- a/qutebrowser/javascript/stylesheet.js +++ b/qutebrowser/javascript/stylesheet.js @@ -99,7 +99,9 @@ window._qutebrowser.stylesheet = (function() { create_style(); return; } - var iter = document.createNodeIterator(document); + var iter = document.createNodeIterator(document, + // eslint-disable-next-line no-bitwise + NodeFilter.SHOW_PROCESSING_INSTRUCTION | NodeFilter.SHOW_ELEMENT); var node; while ((node = iter.nextNode())) { if (check_style(node)) { From 95b41b311f6199da976b7bafbd3148d2aa35287f Mon Sep 17 00:00:00 2001 From: Ulrik de Muelenaere Date: Mon, 30 Oct 2017 22:24:59 +0200 Subject: [PATCH 011/415] Disable ESLint no-bitwise rule --- qutebrowser/javascript/.eslintrc.yaml | 1 + qutebrowser/javascript/stylesheet.js | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/javascript/.eslintrc.yaml b/qutebrowser/javascript/.eslintrc.yaml index d75ef11d8..62ffe52d1 100644 --- a/qutebrowser/javascript/.eslintrc.yaml +++ b/qutebrowser/javascript/.eslintrc.yaml @@ -45,3 +45,4 @@ rules: no-multi-spaces: ["error", {"ignoreEOLComments": true}] function-paren-newline: "off" multiline-comment-style: "off" + no-bitwise: "off" diff --git a/qutebrowser/javascript/stylesheet.js b/qutebrowser/javascript/stylesheet.js index 384138f2f..b1cc70e9f 100644 --- a/qutebrowser/javascript/stylesheet.js +++ b/qutebrowser/javascript/stylesheet.js @@ -100,7 +100,6 @@ window._qutebrowser.stylesheet = (function() { return; } var iter = document.createNodeIterator(document, - // eslint-disable-next-line no-bitwise NodeFilter.SHOW_PROCESSING_INSTRUCTION | NodeFilter.SHOW_ELEMENT); var node; while ((node = iter.nextNode())) { From ce1494e5ec9ecdb93f50eceac1c0a917bf12c80c Mon Sep 17 00:00:00 2001 From: Ulrik de Muelenaere Date: Fri, 3 Nov 2017 12:17:35 +0200 Subject: [PATCH 012/415] Update stylesheet.js to ES6 --- qutebrowser/javascript/stylesheet.js | 64 +++++++++++++--------------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/qutebrowser/javascript/stylesheet.js b/qutebrowser/javascript/stylesheet.js index b1cc70e9f..13741cf41 100644 --- a/qutebrowser/javascript/stylesheet.js +++ b/qutebrowser/javascript/stylesheet.js @@ -24,18 +24,17 @@ window._qutebrowser.stylesheet = (function() { return window._qutebrowser.stylesheet; } - var funcs = {}; + const funcs = {}; - var xhtml_ns = "http://www.w3.org/1999/xhtml"; - var svg_ns = "http://www.w3.org/2000/svg"; + const xhtml_ns = "http://www.w3.org/1999/xhtml"; + const svg_ns = "http://www.w3.org/2000/svg"; - var root_elem; - var style_elem; - var css_content = ""; + let root_elem; + let style_elem; + let css_content = ""; - var root_observer; - var style_observer; - var initialized = false; + let root_observer; + let initialized = false; // Watch for rewrites of the root element and changes to its children, // then move the stylesheet to the end. Partially inspired by Stylus: @@ -53,7 +52,7 @@ window._qutebrowser.stylesheet = (function() { } function create_style() { - var ns = xhtml_ns; + let ns = xhtml_ns; if (document.documentElement.namespaceURI === svg_ns) { ns = svg_ns; } @@ -69,28 +68,15 @@ window._qutebrowser.stylesheet = (function() { // starting point for exploring the relevant code in Chromium, see // https://github.com/qt/qtwebengine-chromium/blob/cfe8c60/chromium/third_party/WebKit/Source/core/xml/parser/XMLDocumentParser.cpp#L1539-L1540 function check_style(node) { - var stylesheet = node.nodeType === Node.PROCESSING_INSTRUCTION_NODE && - node.target === "xml-stylesheet" && - node.parentNode === document; - var known_ns = node.nodeType === Node.ELEMENT_NODE && - (node.namespaceURI === xhtml_ns || - node.namespaceURI === svg_ns); + const stylesheet = node.nodeType === Node.PROCESSING_INSTRUCTION_NODE && + node.target === "xml-stylesheet" && + node.parentNode === document; + const known_ns = node.nodeType === Node.ELEMENT_NODE && + (node.namespaceURI === xhtml_ns || + node.namespaceURI === svg_ns); return stylesheet || known_ns; } - function watch_added_style(mutations) { - for (var mi = 0; mi < mutations.length; ++mi) { - var nodes = mutations[mi].addedNodes; - for (var ni = 0; ni < nodes.length; ++ni) { - if (check_style(nodes[ni])) { - create_style(); - style_observer.disconnect(); - return; - } - } - } - } - function init() { initialized = true; // Chromium will not rewrite a document inside a frame, so add the @@ -99,16 +85,26 @@ window._qutebrowser.stylesheet = (function() { create_style(); return; } - var iter = document.createNodeIterator(document, + const iter = document.createNodeIterator(document, NodeFilter.SHOW_PROCESSING_INSTRUCTION | NodeFilter.SHOW_ELEMENT); - var node; + let node; while ((node = iter.nextNode())) { if (check_style(node)) { create_style(); return; } } - style_observer = new MutationObserver(watch_added_style); + const style_observer = new MutationObserver((mutations) => { + for (const mutation of mutations) { + for (const added of mutation.addedNodes) { + if (check_style(added)) { + create_style(); + style_observer.disconnect(); + return; + } + } + } + }); style_observer.observe(document, {"childList": true, "subtree": true}); } @@ -127,8 +123,8 @@ window._qutebrowser.stylesheet = (function() { } // Propagate the new CSS to all child frames. // FIXME:qtwebengine This does not work for cross-origin frames. - for (var i = 0; i < window.frames.length; ++i) { - var frame = window.frames[i]; + for (let i = 0; i < window.frames.length; ++i) { + const frame = window.frames[i]; if (frame._qutebrowser && frame._qutebrowser.stylesheet) { frame._qutebrowser.stylesheet.set_css(css); } From b37517e55f6363722e576370e5fd6e9af8e007a7 Mon Sep 17 00:00:00 2001 From: Ulrik de Muelenaere Date: Thu, 9 Nov 2017 19:28:36 +0200 Subject: [PATCH 013/415] Fix error in stylesheet.js on older QtWebEngine --- qutebrowser/javascript/stylesheet.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qutebrowser/javascript/stylesheet.js b/qutebrowser/javascript/stylesheet.js index 13741cf41..b1cdeb26e 100644 --- a/qutebrowser/javascript/stylesheet.js +++ b/qutebrowser/javascript/stylesheet.js @@ -96,8 +96,9 @@ window._qutebrowser.stylesheet = (function() { } const style_observer = new MutationObserver((mutations) => { for (const mutation of mutations) { - for (const added of mutation.addedNodes) { - if (check_style(added)) { + const nodes = mutation.addedNodes; + for (let i = 0; i < nodes.length; ++i) { + if (check_style(nodes[i])) { create_style(); style_observer.disconnect(); return; From 2b5e8daba059946fd05050d20364cd00803826eb Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Tue, 7 Nov 2017 11:52:40 -0500 Subject: [PATCH 014/415] Implement qtwebengine version of JSTester --- tests/unit/javascript/conftest.py | 97 +++++++++++++++++++ tests/unit/javascript/stylesheet/simple.html | 4 + .../javascript/stylesheet/test_stylesheet.py | 65 +++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 tests/unit/javascript/stylesheet/simple.html create mode 100644 tests/unit/javascript/stylesheet/test_stylesheet.py diff --git a/tests/unit/javascript/conftest.py b/tests/unit/javascript/conftest.py index b9f013ecf..788daf568 100644 --- a/tests/unit/javascript/conftest.py +++ b/tests/unit/javascript/conftest.py @@ -25,10 +25,12 @@ import logging import pytest import jinja2 +from tests.helpers.fixtures import CallbackChecker try: from PyQt5.QtWebKit import QWebSettings from PyQt5.QtWebKitWidgets import QWebPage + from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineSettings except ImportError: # FIXME:qtwebengine Make these tests use the tab API QWebSettings = None @@ -68,6 +70,34 @@ else: """Fail tests on js console messages as they're used for errors.""" pytest.fail("js console ({}:{}): {}".format(source, line, msg)) + class TestWebEnginePage(QWebEnginePage): + + """QWebPage subclass which overrides some test methods. + + Attributes: + _logger: The logger used for alerts. + """ + + def __init__(self, parent=None): + super().__init__(parent) + self._logger = logging.getLogger('js-tests') + + def javaScriptAlert(self, _frame, msg): + """Log javascript alerts.""" + self._logger.info("js alert: {}".format(msg)) + + def javaScriptConfirm(self, _frame, msg): + """Fail tests on js confirm() as that should never happen.""" + pytest.fail("js confirm: {}".format(msg)) + + def javaScriptPrompt(self, _frame, msg, _default): + """Fail tests on js prompt() as that should never happen.""" + pytest.fail("js prompt: {}".format(msg)) + + def javaScriptConsoleMessage(self, level, msg, line, source): + """Fail tests on js console messages as they're used for errors.""" + pytest.fail("[{}] js console ({}:{}): {}".format(level, source, line, msg)) + class JSTester: @@ -133,8 +163,75 @@ class JSTester: QWebSettings.JavascriptEnabled) return self.webview.page().mainFrame().evaluateJavaScript(source) +class JSWebEngineTester: + + """Object returned by js_tester_webengine which provides test data and a webview. + + Attributes: + webview: The webview which is used. + _qtbot: The QtBot fixture from pytest-qt. + _jinja_env: The jinja2 environment used to get templates. + """ + + def __init__(self, webview, callback_checker, qtbot): + self.webview = webview + self.webview.setPage(TestWebEnginePage(self.webview)) + self.callback_checker = callback_checker + self._qtbot = qtbot + loader = jinja2.FileSystemLoader(os.path.dirname(__file__)) + self._jinja_env = jinja2.Environment(loader=loader, autoescape=True) + + def load(self, path, **kwargs): + """Load and display the given test data. + + Args: + path: The path to the test file, relative to the javascript/ + folder. + **kwargs: Passed to jinja's template.render(). + """ + template = self._jinja_env.get_template(path) + with self._qtbot.waitSignal(self.webview.loadFinished) as blocker: + self.webview.setHtml(template.render(**kwargs)) + assert blocker.args == [True] + + def run_file(self, filename, expected): + """Run a javascript file. + + Args: + filename: The javascript filename, relative to + qutebrowser/javascript. + + Return: + The javascript return value. + """ + source = utils.read_file(os.path.join('javascript', filename)) + self.run(source, expected) + + def run(self, source, expected): + """Run the given javascript source. + + Args: + source: The source to run as a string. + + Return: + The javascript return value. + """ + # TODO how to do this properly + callback_checker = CallbackChecker(self._qtbot) + assert self.webview.settings().testAttribute(QWebEngineSettings.JavascriptEnabled) + self.webview.page().runJavaScript(source, callback_checker.callback) + callback_checker.check(expected) + @pytest.fixture def js_tester(webview, qtbot): """Fixture to test javascript snippets.""" return JSTester(webview, qtbot) + + +@pytest.fixture +def js_tester_webengine(callback_checker, webengineview, qtbot): + """Fixture to test javascript snippets.""" + webengineview.settings().setAttribute( + QWebEngineSettings.JavascriptEnabled, True) + return JSWebEngineTester(webengineview, callback_checker, qtbot) diff --git a/tests/unit/javascript/stylesheet/simple.html b/tests/unit/javascript/stylesheet/simple.html new file mode 100644 index 000000000..4073672a4 --- /dev/null +++ b/tests/unit/javascript/stylesheet/simple.html @@ -0,0 +1,4 @@ +{% extends "base.html" %} +{% block content %} +

Hello World!

+{% endblock %} diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py new file mode 100644 index 000000000..38c1fd116 --- /dev/null +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -0,0 +1,65 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2017 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Tests for position_caret.js.""" + +import os +import pytest +from qutebrowser.utils import javascript +from qutebrowser.browser import shared +from qutebrowser.config import config +from PyQt5.QtWebEngineWidgets import QWebEngineSettings + +class StylesheetTester: + + """Helper class (for the caret_tester fixture) for asserts. + + Attributes: + js: The js_tester fixture. + """ + + def __init__(self, js_tester): + self.js = js_tester + + def check(self): + """Check whether the caret is before the MARKER text.""" + self.js.run('window._qutebrowser = window._qutebrowser || {};', {}) + self.js.run_file('stylesheet.js', {}) + code = javascript.assemble('stylesheet', 'set_css', + "body {background-color: lightblue;}") + self.js.run(code, None) + self.js.run("window.getComputedStyle(document.body, null).getPropertyValue('background-color')", "rgb(173, 216, 230)") + + +@pytest.fixture +@pytest.mark.usefixtures('redirect_webengine_data') +def stylesheet_tester(js_tester_webengine): + """Helper fixture to test caret browsing positions.""" + ss_tester = StylesheetTester(js_tester_webengine) + # Showing webview here is necessary for test_scrolled_down_img to + # succeed in some cases, see #1988 + ss_tester.js.webview.show() + return ss_tester + + +@pytest.mark.integration +def test_simple(stylesheet_tester): + """Test with a simple (one-line) HTML text.""" + stylesheet_tester.js.load('stylesheet/simple.html') + stylesheet_tester.check() From e7fdff56329c0f3de859cf3255b58202d4c0a227 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Wed, 8 Nov 2017 12:27:54 -0500 Subject: [PATCH 015/415] Implement basic stylesheet tests --- tests/helpers/fixtures.py | 5 +- .../javascript/stylesheet/test_stylesheet.py | 46 ++++++++++++++----- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index a01c72788..6fe64a2d2 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -100,7 +100,10 @@ class CallbackChecker(QObject): if self._result is self.UNSET: with self._qtbot.waitSignal(self.got_result, timeout=2000): pass - assert self._result == expected + self._assert_result(self._result, expected) + + def _assert_result(self, result, expected): + assert result == expected @pytest.fixture diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index 38c1fd116..843929eb1 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -26,6 +26,8 @@ from qutebrowser.browser import shared from qutebrowser.config import config from PyQt5.QtWebEngineWidgets import QWebEngineSettings +DEFAULT_BODY_BG = "rgba(0, 0, 0, 0)" + class StylesheetTester: """Helper class (for the caret_tester fixture) for asserts. @@ -37,18 +39,24 @@ class StylesheetTester: def __init__(self, js_tester): self.js = js_tester - def check(self): - """Check whether the caret is before the MARKER text.""" + def init_stylesheet(self): + """Initializes stylesheet. + Run after document is loaded.""" self.js.run('window._qutebrowser = window._qutebrowser || {};', {}) self.js.run_file('stylesheet.js', {}) - code = javascript.assemble('stylesheet', 'set_css', - "body {background-color: lightblue;}") + + def set_css(self, css): + """Set css to CSS via stylesheet.js.""" + code = javascript.assemble('stylesheet', 'set_css', css) self.js.run(code, None) - self.js.run("window.getComputedStyle(document.body, null).getPropertyValue('background-color')", "rgb(173, 216, 230)") + + def check_set(self, element, value): + """Check whether the css in ELEMENT is set to VALUE.""" + self.js.run("window.getComputedStyle(document.body, null)" + ".getPropertyValue('{}');".format(element), value) @pytest.fixture -@pytest.mark.usefixtures('redirect_webengine_data') def stylesheet_tester(js_tester_webengine): """Helper fixture to test caret browsing positions.""" ss_tester = StylesheetTester(js_tester_webengine) @@ -57,9 +65,25 @@ def stylesheet_tester(js_tester_webengine): ss_tester.js.webview.show() return ss_tester - -@pytest.mark.integration -def test_simple(stylesheet_tester): - """Test with a simple (one-line) HTML text.""" +def test_no_set_stylesheet(stylesheet_tester): stylesheet_tester.js.load('stylesheet/simple.html') - stylesheet_tester.check() + stylesheet_tester.init_stylesheet() + stylesheet_tester.check_set("background-color", DEFAULT_BODY_BG) + +def test_no_set_stylesheet_no_load(stylesheet_tester): + stylesheet_tester.js.load('stylesheet/simple.html') + stylesheet_tester.check_set("background-color", DEFAULT_BODY_BG) + +def test_simple_set_bg(stylesheet_tester): + stylesheet_tester.js.load('stylesheet/simple.html') + stylesheet_tester.init_stylesheet() + stylesheet_tester.set_css("body {background-color: rgb(10, 10, 10);}") + stylesheet_tester.check_set("background-color", "rgb(10, 10, 10)") + +def test_simple_set_clear_bg(stylesheet_tester): + stylesheet_tester.js.load('stylesheet/simple.html') + stylesheet_tester.init_stylesheet() + stylesheet_tester.set_css("body {background-color: rgb(10, 10, 10);}") + stylesheet_tester.check_set("background-color", "rgb(10, 10, 10)") + stylesheet_tester.set_css("") + stylesheet_tester.check_set("background-color", DEFAULT_BODY_BG) From 9a1d10ca11070598c7f8c3a72496880f28ee6d0c Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Wed, 8 Nov 2017 12:59:59 -0500 Subject: [PATCH 016/415] Add tests which override existing css --- .../stylesheet/simple_bg_set_red.html | 9 +++++++ .../javascript/stylesheet/test_stylesheet.py | 26 ++++++++++++------- 2 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 tests/unit/javascript/stylesheet/simple_bg_set_red.html diff --git a/tests/unit/javascript/stylesheet/simple_bg_set_red.html b/tests/unit/javascript/stylesheet/simple_bg_set_red.html new file mode 100644 index 000000000..b40352340 --- /dev/null +++ b/tests/unit/javascript/stylesheet/simple_bg_set_red.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} +{% block style %} +body { + background-color: rgb(255, 0, 0); +} +{% endblock %} +{% block content %} +

Hello World!

+{% endblock %} diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index 843929eb1..9057f249b 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -65,17 +65,23 @@ def stylesheet_tester(js_tester_webengine): ss_tester.js.webview.show() return ss_tester -def test_no_set_stylesheet(stylesheet_tester): - stylesheet_tester.js.load('stylesheet/simple.html') - stylesheet_tester.init_stylesheet() - stylesheet_tester.check_set("background-color", DEFAULT_BODY_BG) +@pytest.mark.parametrize('init', [False, True]) +@pytest.mark.parametrize('page,expected', [('stylesheet/simple.html', DEFAULT_BODY_BG), + ('stylesheet/simple_bg_set_red.html', "rgb(255, 0, 0)")]) +def test_no_set_stylesheet(stylesheet_tester, init, page, expected): + stylesheet_tester.js.load(page) + if init: + stylesheet_tester.init_stylesheet() + stylesheet_tester.check_set("background-color", expected) -def test_no_set_stylesheet_no_load(stylesheet_tester): - stylesheet_tester.js.load('stylesheet/simple.html') - stylesheet_tester.check_set("background-color", DEFAULT_BODY_BG) - -def test_simple_set_bg(stylesheet_tester): - stylesheet_tester.js.load('stylesheet/simple.html') +@pytest.mark.parametrize('page', ['stylesheet/simple.html', + 'stylesheet/simple_bg_set_red.html']) +@pytest.mark.parametrize('set_js', [True, False]) +def test_simple_set_bg(stylesheet_tester, page, set_js): + stylesheet_tester.js.load(page) + if set_js: + stylesheet_tester.js.run('document.body.style.backgroundColor = "red";', 'red') + pytest.xfail("overring values set with js does not work.") stylesheet_tester.init_stylesheet() stylesheet_tester.set_css("body {background-color: rgb(10, 10, 10);}") stylesheet_tester.check_set("background-color", "rgb(10, 10, 10)") From 03eae9140e4cfa94a884145d26fafd26dd71d60c Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Wed, 8 Nov 2017 22:58:35 -0500 Subject: [PATCH 017/415] Implement proper loading of stylesheet.js --- tests/unit/javascript/conftest.py | 24 +++++-- tests/unit/javascript/stylesheet/green.css | 1 + tests/unit/javascript/stylesheet/none.css | 0 tests/unit/javascript/stylesheet/simple.xml | 43 ++++++++++++ .../javascript/stylesheet/test_stylesheet.py | 65 ++++++++++--------- 5 files changed, 99 insertions(+), 34 deletions(-) create mode 100644 tests/unit/javascript/stylesheet/green.css create mode 100644 tests/unit/javascript/stylesheet/none.css create mode 100644 tests/unit/javascript/stylesheet/simple.xml diff --git a/tests/unit/javascript/conftest.py b/tests/unit/javascript/conftest.py index 788daf568..25fc9ea9d 100644 --- a/tests/unit/javascript/conftest.py +++ b/tests/unit/javascript/conftest.py @@ -28,9 +28,10 @@ import jinja2 from tests.helpers.fixtures import CallbackChecker try: + from PyQt5.QtCore import QUrl, QFile, QFileInfo from PyQt5.QtWebKit import QWebSettings from PyQt5.QtWebKitWidgets import QWebPage - from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineSettings + from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineSettings, QWebEngineScript except ImportError: # FIXME:qtwebengine Make these tests use the tab API QWebSettings = None @@ -194,6 +195,19 @@ class JSWebEngineTester: self.webview.setHtml(template.render(**kwargs)) assert blocker.args == [True] + def load_file(self, path: str): + """Loads a file from disk""" + self.load_url(QUrl.fromLocalFile( + os.path.join(os.path.dirname(__file__), path))) + + def load_url(self, url: QUrl): + """Load a given QUrl.""" + with self._qtbot.waitSignal(self.webview.loadFinished) as blocker: + self.webview.load(url) + assert blocker.args == [True] + import time + time.sleep(1) + def run_file(self, filename, expected): """Run a javascript file. @@ -207,7 +221,7 @@ class JSWebEngineTester: source = utils.read_file(os.path.join('javascript', filename)) self.run(source, expected) - def run(self, source, expected): + def run(self, source, expected, world=QWebEngineScript.ApplicationWorld): """Run the given javascript source. Args: @@ -218,8 +232,10 @@ class JSWebEngineTester: """ # TODO how to do this properly callback_checker = CallbackChecker(self._qtbot) - assert self.webview.settings().testAttribute(QWebEngineSettings.JavascriptEnabled) - self.webview.page().runJavaScript(source, callback_checker.callback) + assert self.webview.settings().testAttribute( + QWebEngineSettings.JavascriptEnabled) + self.webview.page().runJavaScript(source, world, + callback_checker.callback) callback_checker.check(expected) diff --git a/tests/unit/javascript/stylesheet/green.css b/tests/unit/javascript/stylesheet/green.css new file mode 100644 index 000000000..35832971a --- /dev/null +++ b/tests/unit/javascript/stylesheet/green.css @@ -0,0 +1 @@ +body {background-color: rgb(0, 255, 0);} diff --git a/tests/unit/javascript/stylesheet/none.css b/tests/unit/javascript/stylesheet/none.css new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/javascript/stylesheet/simple.xml b/tests/unit/javascript/stylesheet/simple.xml new file mode 100644 index 000000000..bac4539ab --- /dev/null +++ b/tests/unit/javascript/stylesheet/simple.xml @@ -0,0 +1,43 @@ + + + + org.qutebrowser.qutebrowser + CC-BY-SA-3.0 + GPL-3.0 + qutebrowser + A keyboard-driven web browser + +

+ qutebrowser is a keyboard-focused browser with a minimal GUI. + It was inspired by other browsers/addons like dwb and Vimperator/Pentadactyl, + and is based on Python and PyQt5. +

+ + + Network + WebBrowser + + + qutebrowser + + qutebrowser.desktop + + + https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/main.png + + + https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/downloads.png + + + https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/completion.png + + + https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/hints.png + + + https://www.qutebrowser.org + https://qutebrowser.org/doc/faq.html + https://qutebrowser.org/doc/help/ + https://github.com/qutebrowser/qutebrowser/issues/ + https://github.com/qutebrowser/qutebrowser#donating + diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index 9057f249b..ee557b2a1 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -21,12 +21,16 @@ import os import pytest -from qutebrowser.utils import javascript +from qutebrowser.utils import javascript, utils from qutebrowser.browser import shared from qutebrowser.config import config -from PyQt5.QtWebEngineWidgets import QWebEngineSettings +from PyQt5.QtWebEngineWidgets import QWebEngineSettings, QWebEngineProfile, QWebEngineScript +import qutebrowser.browser.webengine.webenginesettings as webenginesettings DEFAULT_BODY_BG = "rgba(0, 0, 0, 0)" +GREEN_BODY_BG = "rgb(0, 255, 0)" +CSS_BODY_GREEN = "body {background-color: rgb(0, 255, 0);}" +CSS_BODY_RED = "body {background-color: rgb(255, 0, 0);}" class StylesheetTester: @@ -36,60 +40,61 @@ class StylesheetTester: js: The js_tester fixture. """ - def __init__(self, js_tester): + def __init__(self, js_tester, config_stub): self.js = js_tester + self.config_stub = config_stub - def init_stylesheet(self): - """Initializes stylesheet. - Run after document is loaded.""" - self.js.run('window._qutebrowser = window._qutebrowser || {};', {}) - self.js.run_file('stylesheet.js', {}) + def init_stylesheet(self, css_file="green.css"): + + self.config_stub.val.content.user_stylesheets = \ + os.path.join(os.path.dirname(__file__), css_file) + p = QWebEngineProfile.defaultProfile() + webenginesettings._init_stylesheet(p) def set_css(self, css): """Set css to CSS via stylesheet.js.""" code = javascript.assemble('stylesheet', 'set_css', css) self.js.run(code, None) - def check_set(self, element, value): + def check_set(self, value, element="background-color"): """Check whether the css in ELEMENT is set to VALUE.""" self.js.run("window.getComputedStyle(document.body, null)" ".getPropertyValue('{}');".format(element), value) @pytest.fixture -def stylesheet_tester(js_tester_webengine): +def stylesheet_tester(js_tester_webengine, config_stub): """Helper fixture to test caret browsing positions.""" - ss_tester = StylesheetTester(js_tester_webengine) + ss_tester = StylesheetTester(js_tester_webengine, config_stub) # Showing webview here is necessary for test_scrolled_down_img to # succeed in some cases, see #1988 ss_tester.js.webview.show() return ss_tester -@pytest.mark.parametrize('init', [False, True]) -@pytest.mark.parametrize('page,expected', [('stylesheet/simple.html', DEFAULT_BODY_BG), - ('stylesheet/simple_bg_set_red.html', "rgb(255, 0, 0)")]) -def test_no_set_stylesheet(stylesheet_tester, init, page, expected): - stylesheet_tester.js.load(page) - if init: - stylesheet_tester.init_stylesheet() - stylesheet_tester.check_set("background-color", expected) - @pytest.mark.parametrize('page', ['stylesheet/simple.html', 'stylesheet/simple_bg_set_red.html']) @pytest.mark.parametrize('set_js', [True, False]) -def test_simple_set_bg(stylesheet_tester, page, set_js): +def test_set_delayed(stylesheet_tester, page, set_js): + stylesheet_tester.init_stylesheet("none.css") stylesheet_tester.js.load(page) if set_js: - stylesheet_tester.js.run('document.body.style.backgroundColor = "red";', 'red') + stylesheet_tester.js.run( + 'document.body.style.backgroundColor = "red";', 'red') pytest.xfail("overring values set with js does not work.") - stylesheet_tester.init_stylesheet() - stylesheet_tester.set_css("body {background-color: rgb(10, 10, 10);}") - stylesheet_tester.check_set("background-color", "rgb(10, 10, 10)") + stylesheet_tester.set_css("body {background-color: rgb(0, 255, 0);}") + stylesheet_tester.check_set("rgb(0, 255, 0)") -def test_simple_set_clear_bg(stylesheet_tester): - stylesheet_tester.js.load('stylesheet/simple.html') +@pytest.mark.parametrize('page', ['stylesheet/simple.html', + 'stylesheet/simple_bg_set_red.html']) +def test_set_clear_bg(stylesheet_tester, page): stylesheet_tester.init_stylesheet() - stylesheet_tester.set_css("body {background-color: rgb(10, 10, 10);}") - stylesheet_tester.check_set("background-color", "rgb(10, 10, 10)") + stylesheet_tester.js.load('stylesheet/simple.html') + stylesheet_tester.check_set(GREEN_BODY_BG) stylesheet_tester.set_css("") - stylesheet_tester.check_set("background-color", DEFAULT_BODY_BG) + stylesheet_tester.check_set(DEFAULT_BODY_BG) + +def test_no_set_xml(stylesheet_tester): + stylesheet_tester.init_stylesheet() + pytest.xfail("loading xml files throws exceptions") + stylesheet_tester.js.load_file('stylesheet/simple.xml') + stylesheet_tester.check_set(DEFAULT_BODY_BG) From 04b66e1a0ab7592bee41909d0bfa1a4aa15a34c8 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Wed, 8 Nov 2017 23:14:26 -0500 Subject: [PATCH 018/415] Add a test for svg files --- .../javascript/stylesheet/test_stylesheet.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index ee557b2a1..bae5f3e4d 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""Tests for position_caret.js.""" +"""Tests for stylesheet.js.""" import os import pytest @@ -34,10 +34,11 @@ CSS_BODY_RED = "body {background-color: rgb(255, 0, 0);}" class StylesheetTester: - """Helper class (for the caret_tester fixture) for asserts. + """Helper class (for the stylesheet_tester fixture) for asserts. Attributes: js: The js_tester fixture. + config_stub: The config stub object. """ def __init__(self, js_tester, config_stub): @@ -64,10 +65,8 @@ class StylesheetTester: @pytest.fixture def stylesheet_tester(js_tester_webengine, config_stub): - """Helper fixture to test caret browsing positions.""" + """Helper fixture to test stylesheets""" ss_tester = StylesheetTester(js_tester_webengine, config_stub) - # Showing webview here is necessary for test_scrolled_down_img to - # succeed in some cases, see #1988 ss_tester.js.webview.show() return ss_tester @@ -75,6 +74,7 @@ def stylesheet_tester(js_tester_webengine, config_stub): 'stylesheet/simple_bg_set_red.html']) @pytest.mark.parametrize('set_js', [True, False]) def test_set_delayed(stylesheet_tester, page, set_js): + """Test a delayed invocation of set_css.""" stylesheet_tester.init_stylesheet("none.css") stylesheet_tester.js.load(page) if set_js: @@ -87,6 +87,7 @@ def test_set_delayed(stylesheet_tester, page, set_js): @pytest.mark.parametrize('page', ['stylesheet/simple.html', 'stylesheet/simple_bg_set_red.html']) def test_set_clear_bg(stylesheet_tester, page): + """Test setting and clearing the stylesheet""" stylesheet_tester.init_stylesheet() stylesheet_tester.js.load('stylesheet/simple.html') stylesheet_tester.check_set(GREEN_BODY_BG) @@ -94,7 +95,19 @@ def test_set_clear_bg(stylesheet_tester, page): stylesheet_tester.check_set(DEFAULT_BODY_BG) def test_no_set_xml(stylesheet_tester): + """Test stylesheet never modifies xml files.""" stylesheet_tester.init_stylesheet() - pytest.xfail("loading xml files throws exceptions") + pytest.xfail("loading xml/svg files throws exceptions") stylesheet_tester.js.load_file('stylesheet/simple.xml') stylesheet_tester.check_set(DEFAULT_BODY_BG) + stylesheet_tester.set_css("body {background-color: rgb(0, 255, 0);}") + stylesheet_tester.check_set(DEFAULT_BODY_BG) + +def test_no_set_svg(stylesheet_tester): + """Test stylesheet never modifies svg files.""" + stylesheet_tester.init_stylesheet() + pytest.xfail("loading xml/svg files throws exceptions") + stylesheet_tester.js.load_file('../../../misc/cheatsheet.svg') + stylesheet_tester.check_set(None) + stylesheet_tester.set_css("body {background-color: rgb(0, 255, 0);}") + stylesheet_tester.check_set(None) From 2f9a857a27130f7e12ff17f9695e004c42c8ca4e Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Wed, 8 Nov 2017 23:46:12 -0500 Subject: [PATCH 019/415] Add test for styling error pages --- tests/unit/javascript/conftest.py | 9 +++++---- tests/unit/javascript/stylesheet/test_stylesheet.py | 6 ++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/unit/javascript/conftest.py b/tests/unit/javascript/conftest.py index 25fc9ea9d..fa8c48f27 100644 --- a/tests/unit/javascript/conftest.py +++ b/tests/unit/javascript/conftest.py @@ -195,16 +195,17 @@ class JSWebEngineTester: self.webview.setHtml(template.render(**kwargs)) assert blocker.args == [True] - def load_file(self, path: str): + def load_file(self, path: str, force=False): """Loads a file from disk""" self.load_url(QUrl.fromLocalFile( - os.path.join(os.path.dirname(__file__), path))) + os.path.join(os.path.dirname(__file__), path)), force) - def load_url(self, url: QUrl): + def load_url(self, url: QUrl, force=False): """Load a given QUrl.""" with self._qtbot.waitSignal(self.webview.loadFinished) as blocker: self.webview.load(url) - assert blocker.args == [True] + if not force: + assert blocker.args == [True] import time time.sleep(1) diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index bae5f3e4d..736111619 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -111,3 +111,9 @@ def test_no_set_svg(stylesheet_tester): stylesheet_tester.check_set(None) stylesheet_tester.set_css("body {background-color: rgb(0, 255, 0);}") stylesheet_tester.check_set(None) + +def test_set_error(stylesheet_tester): + """Test stylesheet modifies file not found error pages.""" + stylesheet_tester.init_stylesheet() + stylesheet_tester.js.load_file('non-existent.html', force=True) + stylesheet_tester.check_set(GREEN_BODY_BG) From 5ac8e5ad3e15974a848ccd05e8cde9c9dd18fa7a Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Thu, 9 Nov 2017 00:07:54 -0500 Subject: [PATCH 020/415] Clean up stylesheet tests --- tests/unit/javascript/conftest.py | 64 ++++++++++++------- tests/unit/javascript/stylesheet/green.css | 2 +- tests/unit/javascript/stylesheet/simple.xml | 38 ----------- .../javascript/stylesheet/test_stylesheet.py | 55 ++++++++-------- 4 files changed, 71 insertions(+), 88 deletions(-) diff --git a/tests/unit/javascript/conftest.py b/tests/unit/javascript/conftest.py index fa8c48f27..b6b0980d5 100644 --- a/tests/unit/javascript/conftest.py +++ b/tests/unit/javascript/conftest.py @@ -26,17 +26,25 @@ import logging import pytest import jinja2 from tests.helpers.fixtures import CallbackChecker +from PyQt5.QtCore import QUrl try: - from PyQt5.QtCore import QUrl, QFile, QFileInfo from PyQt5.QtWebKit import QWebSettings from PyQt5.QtWebKitWidgets import QWebPage - from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineSettings, QWebEngineScript except ImportError: # FIXME:qtwebengine Make these tests use the tab API QWebSettings = None QWebPage = None +try: + from PyQt5.QtWebEngineWidgets import (QWebEnginePage, + QWebEngineSettings, + QWebEngineScript) +except ImportError: + QWebEnginePage = None + QWebEngineSettings = None + QWebEngineScript = None + from qutebrowser.utils import utils @@ -71,9 +79,12 @@ else: """Fail tests on js console messages as they're used for errors.""" pytest.fail("js console ({}:{}): {}".format(source, line, msg)) +if QWebEnginePage is None: + TestWebEnginePage = None +else: class TestWebEnginePage(QWebEnginePage): - """QWebPage subclass which overrides some test methods. + """QWebEnginePage which overrides javascript logging methods. Attributes: _logger: The logger used for alerts. @@ -97,7 +108,8 @@ else: def javaScriptConsoleMessage(self, level, msg, line, source): """Fail tests on js console messages as they're used for errors.""" - pytest.fail("[{}] js console ({}:{}): {}".format(level, source, line, msg)) + pytest.fail("[{}] js console ({}:{}): {}".format(level, source, + line, msg)) class JSTester: @@ -164,9 +176,10 @@ class JSTester: QWebSettings.JavascriptEnabled) return self.webview.page().mainFrame().evaluateJavaScript(source) + class JSWebEngineTester: - """Object returned by js_tester_webengine which provides test data and a webview. + """Object returned by js_tester_webengine which provides a webview. Attributes: webview: The webview which is used. @@ -183,7 +196,7 @@ class JSWebEngineTester: self._jinja_env = jinja2.Environment(loader=loader, autoescape=True) def load(self, path, **kwargs): - """Load and display the given test data. + """Load and display the given jinja test data. Args: path: The path to the test file, relative to the javascript/ @@ -196,42 +209,47 @@ class JSWebEngineTester: assert blocker.args == [True] def load_file(self, path: str, force=False): - """Loads a file from disk""" + """Load a file from disk. + + Args: + path: The string path from disk to load (relative to this file) + force: Whether to force loading even if the file is invalid. + """ self.load_url(QUrl.fromLocalFile( os.path.join(os.path.dirname(__file__), path)), force) - def load_url(self, url: QUrl, force=False): - """Load a given QUrl.""" + def load_url(self, url: QUrl, force: bool = False): + """Load a given QUrl. + + Args: + url: The QUrl to load. + force: Whether to force loading even if the file is invalid. + """ with self._qtbot.waitSignal(self.webview.loadFinished) as blocker: self.webview.load(url) if not force: assert blocker.args == [True] - import time - time.sleep(1) - def run_file(self, filename, expected): + def run_file(self, filename: str, expected) -> None: """Run a javascript file. Args: filename: The javascript filename, relative to qutebrowser/javascript. - - Return: - The javascript return value. + expected: The value expected return from the javascript execution """ source = utils.read_file(os.path.join('javascript', filename)) self.run(source, expected) - def run(self, source, expected, world=QWebEngineScript.ApplicationWorld): + def run(self, source: str, expected, + world=QWebEngineScript.ApplicationWorld) -> None: """Run the given javascript source. Args: source: The source to run as a string. - - Return: - The javascript return value. + expected: The value expected return from the javascript execution + world: The scope the javascript will run in """ - # TODO how to do this properly callback_checker = CallbackChecker(self._qtbot) assert self.webview.settings().testAttribute( QWebEngineSettings.JavascriptEnabled) @@ -242,13 +260,11 @@ class JSWebEngineTester: @pytest.fixture def js_tester(webview, qtbot): - """Fixture to test javascript snippets.""" + """Fixture to test javascript snippets in webkit.""" return JSTester(webview, qtbot) @pytest.fixture def js_tester_webengine(callback_checker, webengineview, qtbot): - """Fixture to test javascript snippets.""" - webengineview.settings().setAttribute( - QWebEngineSettings.JavascriptEnabled, True) + """Fixture to test javascript snippets in webengine.""" return JSWebEngineTester(webengineview, callback_checker, qtbot) diff --git a/tests/unit/javascript/stylesheet/green.css b/tests/unit/javascript/stylesheet/green.css index 35832971a..b2d035810 100644 --- a/tests/unit/javascript/stylesheet/green.css +++ b/tests/unit/javascript/stylesheet/green.css @@ -1 +1 @@ -body {background-color: rgb(0, 255, 0);} +body, :root {background-color: rgb(0, 255, 0);} diff --git a/tests/unit/javascript/stylesheet/simple.xml b/tests/unit/javascript/stylesheet/simple.xml index bac4539ab..f9073de69 100644 --- a/tests/unit/javascript/stylesheet/simple.xml +++ b/tests/unit/javascript/stylesheet/simple.xml @@ -2,42 +2,4 @@ org.qutebrowser.qutebrowser - CC-BY-SA-3.0 - GPL-3.0 - qutebrowser - A keyboard-driven web browser - -

- qutebrowser is a keyboard-focused browser with a minimal GUI. - It was inspired by other browsers/addons like dwb and Vimperator/Pentadactyl, - and is based on Python and PyQt5. -

- - - Network - WebBrowser - - - qutebrowser - - qutebrowser.desktop - - - https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/main.png - - - https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/downloads.png - - - https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/completion.png - - - https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/hints.png - - - https://www.qutebrowser.org - https://qutebrowser.org/doc/faq.html - https://qutebrowser.org/doc/help/ - https://github.com/qutebrowser/qutebrowser/issues/ - https://github.com/qutebrowser/qutebrowser#donating diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index 736111619..9be7192e9 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -1,6 +1,6 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# Copyright 2015-2017 Florian Bruhin (The Compiler) +# Copyright 2017 Jay Kamat # # This file is part of qutebrowser. # @@ -21,17 +21,17 @@ import os import pytest -from qutebrowser.utils import javascript, utils -from qutebrowser.browser import shared -from qutebrowser.config import config -from PyQt5.QtWebEngineWidgets import QWebEngineSettings, QWebEngineProfile, QWebEngineScript -import qutebrowser.browser.webengine.webenginesettings as webenginesettings +from qutebrowser.utils import javascript +from PyQt5.QtWebEngineWidgets import QWebEngineProfile +from qutebrowser.browser.webengine import webenginesettings + DEFAULT_BODY_BG = "rgba(0, 0, 0, 0)" GREEN_BODY_BG = "rgb(0, 255, 0)" CSS_BODY_GREEN = "body {background-color: rgb(0, 255, 0);}" CSS_BODY_RED = "body {background-color: rgb(255, 0, 0);}" + class StylesheetTester: """Helper class (for the stylesheet_tester fixture) for asserts. @@ -46,71 +46,76 @@ class StylesheetTester: self.config_stub = config_stub def init_stylesheet(self, css_file="green.css"): - - self.config_stub.val.content.user_stylesheets = \ - os.path.join(os.path.dirname(__file__), css_file) + """Initialize the stylesheet with a provided css file.""" + css_path = os.path.join(os.path.dirname(__file__), css_file) + self.config_stub.val.content.user_stylesheets = css_path p = QWebEngineProfile.defaultProfile() webenginesettings._init_stylesheet(p) def set_css(self, css): - """Set css to CSS via stylesheet.js.""" + """Set document style to `css` via stylesheet.js.""" code = javascript.assemble('stylesheet', 'set_css', css) self.js.run(code, None) - def check_set(self, value, element="background-color"): + def check_set(self, value, css_style="background-color", + document_element="document.body"): """Check whether the css in ELEMENT is set to VALUE.""" - self.js.run("window.getComputedStyle(document.body, null)" - ".getPropertyValue('{}');".format(element), value) + self.js.run("window.getComputedStyle({}, null)" + ".getPropertyValue('{}');".format(document_element, + css_style), value) @pytest.fixture def stylesheet_tester(js_tester_webengine, config_stub): - """Helper fixture to test stylesheets""" + """Helper fixture to test stylesheets.""" ss_tester = StylesheetTester(js_tester_webengine, config_stub) ss_tester.js.webview.show() return ss_tester + @pytest.mark.parametrize('page', ['stylesheet/simple.html', 'stylesheet/simple_bg_set_red.html']) -@pytest.mark.parametrize('set_js', [True, False]) -def test_set_delayed(stylesheet_tester, page, set_js): +def test_set_delayed(stylesheet_tester, page): """Test a delayed invocation of set_css.""" stylesheet_tester.init_stylesheet("none.css") stylesheet_tester.js.load(page) - if set_js: - stylesheet_tester.js.run( - 'document.body.style.backgroundColor = "red";', 'red') - pytest.xfail("overring values set with js does not work.") stylesheet_tester.set_css("body {background-color: rgb(0, 255, 0);}") stylesheet_tester.check_set("rgb(0, 255, 0)") + @pytest.mark.parametrize('page', ['stylesheet/simple.html', 'stylesheet/simple_bg_set_red.html']) def test_set_clear_bg(stylesheet_tester, page): - """Test setting and clearing the stylesheet""" + """Test setting and clearing the stylesheet.""" stylesheet_tester.init_stylesheet() stylesheet_tester.js.load('stylesheet/simple.html') stylesheet_tester.check_set(GREEN_BODY_BG) stylesheet_tester.set_css("") stylesheet_tester.check_set(DEFAULT_BODY_BG) + def test_no_set_xml(stylesheet_tester): """Test stylesheet never modifies xml files.""" stylesheet_tester.init_stylesheet() - pytest.xfail("loading xml/svg files throws exceptions") stylesheet_tester.js.load_file('stylesheet/simple.xml') + pytest.xfail("stylesheet is set on xml documents") stylesheet_tester.check_set(DEFAULT_BODY_BG) stylesheet_tester.set_css("body {background-color: rgb(0, 255, 0);}") stylesheet_tester.check_set(DEFAULT_BODY_BG) + def test_no_set_svg(stylesheet_tester): """Test stylesheet never modifies svg files.""" stylesheet_tester.init_stylesheet() - pytest.xfail("loading xml/svg files throws exceptions") + # pytest.xfail("loading xml/svg files throws exceptions") stylesheet_tester.js.load_file('../../../misc/cheatsheet.svg') - stylesheet_tester.check_set(None) + pytest.xfail("stylesheet is set on svg documents??") + stylesheet_tester.check_set(DEFAULT_BODY_BG, + document_element="document.documentElement") stylesheet_tester.set_css("body {background-color: rgb(0, 255, 0);}") - stylesheet_tester.check_set(None) + stylesheet_tester.check_set(DEFAULT_BODY_BG, + document_element="document.documentElement") + def test_set_error(stylesheet_tester): """Test stylesheet modifies file not found error pages.""" From 155ee198cdaa14972bab06656865b9ef7ad2685c Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Thu, 9 Nov 2017 14:44:14 -0500 Subject: [PATCH 021/415] Update stylesheet tests for updates in stylesheet.js --- .../javascript/stylesheet/test_stylesheet.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index 9be7192e9..4a6a06c2c 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -64,6 +64,10 @@ class StylesheetTester: ".getPropertyValue('{}');".format(document_element, css_style), value) + def check_eq(self, one, two, true=True): + """Check if one and two are equal.""" + self.js.run("{} === {}".format(one, two), true) + @pytest.fixture def stylesheet_tester(js_tester_webengine, config_stub): @@ -94,27 +98,20 @@ def test_set_clear_bg(stylesheet_tester, page): stylesheet_tester.check_set(DEFAULT_BODY_BG) -def test_no_set_xml(stylesheet_tester): - """Test stylesheet never modifies xml files.""" +def test_set_xml(stylesheet_tester): + """Test stylesheet is applied without altering xml files.""" stylesheet_tester.init_stylesheet() stylesheet_tester.js.load_file('stylesheet/simple.xml') - pytest.xfail("stylesheet is set on xml documents") - stylesheet_tester.check_set(DEFAULT_BODY_BG) - stylesheet_tester.set_css("body {background-color: rgb(0, 255, 0);}") - stylesheet_tester.check_set(DEFAULT_BODY_BG) + stylesheet_tester.check_set(GREEN_BODY_BG) + stylesheet_tester.check_eq("\"html\"", "document.documentElement.nodeName") - -def test_no_set_svg(stylesheet_tester): - """Test stylesheet never modifies svg files.""" +def test_set_svg(stylesheet_tester): + """Test stylesheet is applied for svg files.""" stylesheet_tester.init_stylesheet() - # pytest.xfail("loading xml/svg files throws exceptions") stylesheet_tester.js.load_file('../../../misc/cheatsheet.svg') - pytest.xfail("stylesheet is set on svg documents??") - stylesheet_tester.check_set(DEFAULT_BODY_BG, - document_element="document.documentElement") - stylesheet_tester.set_css("body {background-color: rgb(0, 255, 0);}") - stylesheet_tester.check_set(DEFAULT_BODY_BG, + stylesheet_tester.check_set(GREEN_BODY_BG, document_element="document.documentElement") + stylesheet_tester.check_eq("\"svg\"", "document.documentElement.nodeName") def test_set_error(stylesheet_tester): From d39dda38cef502cbda6a572284c30714bdffa5b0 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Fri, 10 Nov 2017 12:27:00 -0500 Subject: [PATCH 022/415] Refactor CallbackChecker into test utils --- tests/helpers/fixtures.py | 31 ++----------------------------- tests/helpers/utils.py | 26 ++++++++++++++++++++++++++ tests/unit/javascript/conftest.py | 2 +- 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 6fe64a2d2..8caf6fcf5 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -37,13 +37,14 @@ import pytest import py.path # pylint: disable=no-name-in-module import helpers.stubs as stubsmod +from helpers.utils import CallbackChecker from qutebrowser.config import config, configdata, configtypes, configexc from qutebrowser.utils import objreg, standarddir from qutebrowser.browser.webkit import cookies from qutebrowser.misc import savemanager, sql from qutebrowser.keyinput import modeman -from PyQt5.QtCore import pyqtSignal, QEvent, QSize, Qt, QObject +from PyQt5.QtCore import QEvent, QSize, Qt from PyQt5.QtGui import QKeyEvent from PyQt5.QtWidgets import QWidget, QHBoxLayout, QVBoxLayout from PyQt5.QtNetwork import QNetworkCookieJar @@ -78,34 +79,6 @@ class WinRegistryHelper: del objreg.window_registry[win_id] -class CallbackChecker(QObject): - - """Check if a value provided by a callback is the expected one.""" - - got_result = pyqtSignal(object) - UNSET = object() - - def __init__(self, qtbot, parent=None): - super().__init__(parent) - self._qtbot = qtbot - self._result = self.UNSET - - def callback(self, result): - """Callback which can be passed to runJavaScript.""" - self._result = result - self.got_result.emit(result) - - def check(self, expected): - """Wait until the JS result arrived and compare it.""" - if self._result is self.UNSET: - with self._qtbot.waitSignal(self.got_result, timeout=2000): - pass - self._assert_result(self._result, expected) - - def _assert_result(self, result, expected): - assert result == expected - - @pytest.fixture def callback_checker(qtbot): return CallbackChecker(qtbot) diff --git a/tests/helpers/utils.py b/tests/helpers/utils.py index e6e3d37c8..45141f7dc 100644 --- a/tests/helpers/utils.py +++ b/tests/helpers/utils.py @@ -28,6 +28,7 @@ import contextlib import pytest from qutebrowser.utils import qtutils +from PyQt5.QtCore import QObject, pyqtSignal qt58 = pytest.mark.skipif( @@ -176,3 +177,28 @@ def abs_datapath(): @contextlib.contextmanager def nop_contextmanager(): yield + + +class CallbackChecker(QObject): + + """Check if a value provided by a callback is the expected one.""" + + got_result = pyqtSignal(object) + UNSET = object() + + def __init__(self, qtbot, parent=None): + super().__init__(parent) + self._qtbot = qtbot + self._result = self.UNSET + + def callback(self, result): + """Callback which can be passed to runJavaScript.""" + self._result = result + self.got_result.emit(result) + + def check(self, expected): + """Wait until the JS result arrived and compare it.""" + if self._result is self.UNSET: + with self._qtbot.waitSignal(self.got_result, timeout=2000): + pass + assert self._result == expected diff --git a/tests/unit/javascript/conftest.py b/tests/unit/javascript/conftest.py index b6b0980d5..103c9cf4d 100644 --- a/tests/unit/javascript/conftest.py +++ b/tests/unit/javascript/conftest.py @@ -25,7 +25,7 @@ import logging import pytest import jinja2 -from tests.helpers.fixtures import CallbackChecker +from helpers.utils import CallbackChecker from PyQt5.QtCore import QUrl try: From 83e28a70c5a2cbfd2ed51516e76407ae316cfbab Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Fri, 10 Nov 2017 12:34:23 -0500 Subject: [PATCH 023/415] Fix error message printing for webengine js tests --- tests/unit/javascript/conftest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/javascript/conftest.py b/tests/unit/javascript/conftest.py index 103c9cf4d..413609b32 100644 --- a/tests/unit/javascript/conftest.py +++ b/tests/unit/javascript/conftest.py @@ -27,6 +27,7 @@ import pytest import jinja2 from helpers.utils import CallbackChecker from PyQt5.QtCore import QUrl +from qutebrowser.utils.debug import qenum_key try: from PyQt5.QtWebKit import QWebSettings @@ -108,8 +109,8 @@ else: def javaScriptConsoleMessage(self, level, msg, line, source): """Fail tests on js console messages as they're used for errors.""" - pytest.fail("[{}] js console ({}:{}): {}".format(level, source, - line, msg)) + pytest.fail("[{}] js console ({}:{}): {}".format( + qenum_key(QWebEnginePage, level), source, line, msg)) class JSTester: From 324c537a3df65396cf7e1b217bcaa14874620d42 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Fri, 10 Nov 2017 12:55:27 -0500 Subject: [PATCH 024/415] Refactor webkit and webengine js testers to have a common subclass --- tests/unit/javascript/conftest.py | 59 +++++++++---------- .../javascript/stylesheet/test_stylesheet.py | 1 + 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/tests/unit/javascript/conftest.py b/tests/unit/javascript/conftest.py index 413609b32..bcfc6b47a 100644 --- a/tests/unit/javascript/conftest.py +++ b/tests/unit/javascript/conftest.py @@ -114,8 +114,7 @@ else: class JSTester: - - """Object returned by js_tester which provides test data and a webview. + """Common subclass providing basic functionality for all JS testers. Attributes: webview: The webview which is used. @@ -125,21 +124,12 @@ class JSTester: def __init__(self, webview, qtbot): self.webview = webview - self.webview.setPage(TestWebPage(self.webview)) self._qtbot = qtbot loader = jinja2.FileSystemLoader(os.path.dirname(__file__)) self._jinja_env = jinja2.Environment(loader=loader, autoescape=True) - def scroll_anchor(self, name): - """Scroll the main frame to the given anchor.""" - page = self.webview.page() - old_pos = page.mainFrame().scrollPosition() - page.mainFrame().scrollToAnchor(name) - new_pos = page.mainFrame().scrollPosition() - assert old_pos != new_pos - def load(self, path, **kwargs): - """Load and display the given test data. + """Load and display the given jinja test data. Args: path: The path to the test file, relative to the javascript/ @@ -151,6 +141,29 @@ class JSTester: self.webview.setHtml(template.render(**kwargs)) assert blocker.args == [True] + +class JSWebKitTester(JSTester): + + """Object returned by js_tester which provides test data and a webview. + + Attributes: + webview: The webview which is used. + _qtbot: The QtBot fixture from pytest-qt. + _jinja_env: The jinja2 environment used to get templates. + """ + + def __init__(self, webview, qtbot): + JSTester.__init__(self, webview, qtbot) + self.webview.setPage(TestWebPage(self.webview)) + + def scroll_anchor(self, name): + """Scroll the main frame to the given anchor.""" + page = self.webview.page() + old_pos = page.mainFrame().scrollPosition() + page.mainFrame().scrollToAnchor(name) + new_pos = page.mainFrame().scrollPosition() + assert old_pos != new_pos + def run_file(self, filename): """Run a javascript file. @@ -178,7 +191,7 @@ class JSTester: return self.webview.page().mainFrame().evaluateJavaScript(source) -class JSWebEngineTester: +class JSWebEngineTester(JSTester): """Object returned by js_tester_webengine which provides a webview. @@ -189,25 +202,9 @@ class JSWebEngineTester: """ def __init__(self, webview, callback_checker, qtbot): - self.webview = webview + JSTester.__init__(self, webview, qtbot) self.webview.setPage(TestWebEnginePage(self.webview)) self.callback_checker = callback_checker - self._qtbot = qtbot - loader = jinja2.FileSystemLoader(os.path.dirname(__file__)) - self._jinja_env = jinja2.Environment(loader=loader, autoescape=True) - - def load(self, path, **kwargs): - """Load and display the given jinja test data. - - Args: - path: The path to the test file, relative to the javascript/ - folder. - **kwargs: Passed to jinja's template.render(). - """ - template = self._jinja_env.get_template(path) - with self._qtbot.waitSignal(self.webview.loadFinished) as blocker: - self.webview.setHtml(template.render(**kwargs)) - assert blocker.args == [True] def load_file(self, path: str, force=False): """Load a file from disk. @@ -262,7 +259,7 @@ class JSWebEngineTester: @pytest.fixture def js_tester(webview, qtbot): """Fixture to test javascript snippets in webkit.""" - return JSTester(webview, qtbot) + return JSWebKitTester(webview, qtbot) @pytest.fixture diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index 4a6a06c2c..179240d42 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -105,6 +105,7 @@ def test_set_xml(stylesheet_tester): stylesheet_tester.check_set(GREEN_BODY_BG) stylesheet_tester.check_eq("\"html\"", "document.documentElement.nodeName") + def test_set_svg(stylesheet_tester): """Test stylesheet is applied for svg files.""" stylesheet_tester.init_stylesheet() From 5913552dfec4f6e1a812a558cb54b3ca386de1f3 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Mon, 13 Nov 2017 19:57:11 -0500 Subject: [PATCH 025/415] Fix style issues in stylesheet tests --- tests/helpers/fixtures.py | 4 +- tests/helpers/utils.py | 3 +- tests/unit/javascript/conftest.py | 76 ++++++++++--------- .../position_caret/test_position_caret.py | 4 +- .../javascript/stylesheet/test_stylesheet.py | 15 ++-- 5 files changed, 55 insertions(+), 47 deletions(-) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 8caf6fcf5..6e17ebd71 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -37,7 +37,7 @@ import pytest import py.path # pylint: disable=no-name-in-module import helpers.stubs as stubsmod -from helpers.utils import CallbackChecker +import helpers.utils from qutebrowser.config import config, configdata, configtypes, configexc from qutebrowser.utils import objreg, standarddir from qutebrowser.browser.webkit import cookies @@ -81,7 +81,7 @@ class WinRegistryHelper: @pytest.fixture def callback_checker(qtbot): - return CallbackChecker(qtbot) + return helpers.utils.CallbackChecker(qtbot) class FakeStatusBar(QWidget): diff --git a/tests/helpers/utils.py b/tests/helpers/utils.py index 45141f7dc..82c07fbd2 100644 --- a/tests/helpers/utils.py +++ b/tests/helpers/utils.py @@ -27,9 +27,10 @@ import contextlib import pytest -from qutebrowser.utils import qtutils from PyQt5.QtCore import QObject, pyqtSignal +from qutebrowser.utils import qtutils + qt58 = pytest.mark.skipif( qtutils.version_check('5.9'), reason="Needs Qt 5.8 or earlier") diff --git a/tests/unit/javascript/conftest.py b/tests/unit/javascript/conftest.py index bcfc6b47a..7290d4d00 100644 --- a/tests/unit/javascript/conftest.py +++ b/tests/unit/javascript/conftest.py @@ -25,9 +25,12 @@ import logging import pytest import jinja2 -from helpers.utils import CallbackChecker + from PyQt5.QtCore import QUrl -from qutebrowser.utils.debug import qenum_key + +import helpers.utils +import qutebrowser.utils.debug +from qutebrowser.utils import utils try: from PyQt5.QtWebKit import QWebSettings @@ -46,8 +49,6 @@ except ImportError: QWebEngineSettings = None QWebEngineScript = None -from qutebrowser.utils import utils - if QWebPage is None: TestWebPage = None @@ -110,10 +111,12 @@ else: def javaScriptConsoleMessage(self, level, msg, line, source): """Fail tests on js console messages as they're used for errors.""" pytest.fail("[{}] js console ({}:{}): {}".format( - qenum_key(QWebEnginePage, level), source, line, msg)) + qutebrowser.utils.debug.qenum_key( + QWebEnginePage, level), source, line, msg)) class JSTester: + """Common subclass providing basic functionality for all JS testers. Attributes: @@ -141,6 +144,28 @@ class JSTester: self.webview.setHtml(template.render(**kwargs)) assert blocker.args == [True] + def load_file(self, path: str, force: bool = False): + """Load a file from disk. + + Args: + path: The string path from disk to load (relative to this file) + force: Whether to force loading even if the file is invalid. + """ + self.load_url(QUrl.fromLocalFile( + os.path.join(os.path.dirname(__file__), path)), force) + + def load_url(self, url: QUrl, force: bool = False): + """Load a given QUrl. + + Args: + url: The QUrl to load. + force: Whether to force loading even if the file is invalid. + """ + with self._qtbot.waitSignal(self.webview.loadFinished) as blocker: + self.webview.load(url) + if not force: + assert blocker.args == [True] + class JSWebKitTester(JSTester): @@ -153,7 +178,7 @@ class JSWebKitTester(JSTester): """ def __init__(self, webview, qtbot): - JSTester.__init__(self, webview, qtbot) + super().__init__(webview, qtbot) self.webview.setPage(TestWebPage(self.webview)) def scroll_anchor(self, name): @@ -201,32 +226,9 @@ class JSWebEngineTester(JSTester): _jinja_env: The jinja2 environment used to get templates. """ - def __init__(self, webview, callback_checker, qtbot): - JSTester.__init__(self, webview, qtbot) + def __init__(self, webview, qtbot): + super().__init__(webview, qtbot) self.webview.setPage(TestWebEnginePage(self.webview)) - self.callback_checker = callback_checker - - def load_file(self, path: str, force=False): - """Load a file from disk. - - Args: - path: The string path from disk to load (relative to this file) - force: Whether to force loading even if the file is invalid. - """ - self.load_url(QUrl.fromLocalFile( - os.path.join(os.path.dirname(__file__), path)), force) - - def load_url(self, url: QUrl, force: bool = False): - """Load a given QUrl. - - Args: - url: The QUrl to load. - force: Whether to force loading even if the file is invalid. - """ - with self._qtbot.waitSignal(self.webview.loadFinished) as blocker: - self.webview.load(url) - if not force: - assert blocker.args == [True] def run_file(self, filename: str, expected) -> None: """Run a javascript file. @@ -239,8 +241,7 @@ class JSWebEngineTester(JSTester): source = utils.read_file(os.path.join('javascript', filename)) self.run(source, expected) - def run(self, source: str, expected, - world=QWebEngineScript.ApplicationWorld) -> None: + def run(self, source: str, expected, world=None) -> None: """Run the given javascript source. Args: @@ -248,7 +249,10 @@ class JSWebEngineTester(JSTester): expected: The value expected return from the javascript execution world: The scope the javascript will run in """ - callback_checker = CallbackChecker(self._qtbot) + if world is None: + world = QWebEngineScript.ApplicationWorld + + callback_checker = helpers.utils.CallbackChecker(self._qtbot) assert self.webview.settings().testAttribute( QWebEngineSettings.JavascriptEnabled) self.webview.page().runJavaScript(source, world, @@ -257,7 +261,7 @@ class JSWebEngineTester(JSTester): @pytest.fixture -def js_tester(webview, qtbot): +def js_tester_webkit(webview, qtbot): """Fixture to test javascript snippets in webkit.""" return JSWebKitTester(webview, qtbot) @@ -265,4 +269,4 @@ def js_tester(webview, qtbot): @pytest.fixture def js_tester_webengine(callback_checker, webengineview, qtbot): """Fixture to test javascript snippets in webengine.""" - return JSWebEngineTester(webengineview, callback_checker, qtbot) + return JSWebEngineTester(webengineview, qtbot) diff --git a/tests/unit/javascript/position_caret/test_position_caret.py b/tests/unit/javascript/position_caret/test_position_caret.py index 7be62e3cc..fcfa5cf5d 100644 --- a/tests/unit/javascript/position_caret/test_position_caret.py +++ b/tests/unit/javascript/position_caret/test_position_caret.py @@ -65,9 +65,9 @@ class CaretTester: @pytest.fixture -def caret_tester(js_tester): +def caret_tester(js_tester_webkit): """Helper fixture to test caret browsing positions.""" - caret_tester = CaretTester(js_tester) + caret_tester = CaretTester(js_tester_webkit) # Showing webview here is necessary for test_scrolled_down_img to # succeed in some cases, see #1988 caret_tester.js.webview.show() diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index 179240d42..f74c04b8e 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -21,8 +21,10 @@ import os import pytest -from qutebrowser.utils import javascript + from PyQt5.QtWebEngineWidgets import QWebEngineProfile + +from qutebrowser.utils import javascript from qutebrowser.browser.webengine import webenginesettings @@ -61,12 +63,13 @@ class StylesheetTester: document_element="document.body"): """Check whether the css in ELEMENT is set to VALUE.""" self.js.run("window.getComputedStyle({}, null)" - ".getPropertyValue('{}');".format(document_element, - css_style), value) + ".getPropertyValue('{}');" + .format(document_element, + javascript.string_escape(css_style)), value) def check_eq(self, one, two, true=True): """Check if one and two are equal.""" - self.js.run("{} === {}".format(one, two), true) + self.js.run("{} === {};".format(one, two), true) @pytest.fixture @@ -103,7 +106,7 @@ def test_set_xml(stylesheet_tester): stylesheet_tester.init_stylesheet() stylesheet_tester.js.load_file('stylesheet/simple.xml') stylesheet_tester.check_set(GREEN_BODY_BG) - stylesheet_tester.check_eq("\"html\"", "document.documentElement.nodeName") + stylesheet_tester.check_eq('"html"', "document.documentElement.nodeName") def test_set_svg(stylesheet_tester): @@ -112,7 +115,7 @@ def test_set_svg(stylesheet_tester): stylesheet_tester.js.load_file('../../../misc/cheatsheet.svg') stylesheet_tester.check_set(GREEN_BODY_BG, document_element="document.documentElement") - stylesheet_tester.check_eq("\"svg\"", "document.documentElement.nodeName") + stylesheet_tester.check_eq('"svg"', "document.documentElement.nodeName") def test_set_error(stylesheet_tester): From 28572ce3b1d755e91d648c1cfae0df8e5b966acb Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Mon, 13 Nov 2017 22:08:06 -0500 Subject: [PATCH 026/415] Fix stylesheet tests crashing when no QtWebEngine available --- .../unit/javascript/stylesheet/test_stylesheet.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index f74c04b8e..83c9cbf98 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -22,10 +22,18 @@ import os import pytest -from PyQt5.QtWebEngineWidgets import QWebEngineProfile +try: + from PyQt5.QtWebEngineWidgets import QWebEngineProfile +except ImportError: + QWebEngineProfile = None + from qutebrowser.utils import javascript -from qutebrowser.browser.webengine import webenginesettings + +try: + from qutebrowser.browser.webengine import webenginesettings +except ImportError: + webenginesettings = None DEFAULT_BODY_BG = "rgba(0, 0, 0, 0)" @@ -51,6 +59,8 @@ class StylesheetTester: """Initialize the stylesheet with a provided css file.""" css_path = os.path.join(os.path.dirname(__file__), css_file) self.config_stub.val.content.user_stylesheets = css_path + if QWebEngineProfile is None: + pytest.skip("QTWebEngine not found.") p = QWebEngineProfile.defaultProfile() webenginesettings._init_stylesheet(p) From 92a6e61b5226ed25a6d133920c4b9f0e4418bc47 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Wed, 15 Nov 2017 02:05:34 -0500 Subject: [PATCH 027/415] Use importorskip to skip stylesheet tests on webkit only systems --- tests/unit/javascript/stylesheet/test_stylesheet.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index 83c9cbf98..7070cfceb 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -22,11 +22,8 @@ import os import pytest -try: - from PyQt5.QtWebEngineWidgets import QWebEngineProfile -except ImportError: - QWebEngineProfile = None - +QtWebEngineWidgets = pytest.importorskip("PyQt5.QtWebEngineWidgets") +QWebEngineProfile = QtWebEngineWidgets.QWebEngineProfile from qutebrowser.utils import javascript @@ -59,8 +56,6 @@ class StylesheetTester: """Initialize the stylesheet with a provided css file.""" css_path = os.path.join(os.path.dirname(__file__), css_file) self.config_stub.val.content.user_stylesheets = css_path - if QWebEngineProfile is None: - pytest.skip("QTWebEngine not found.") p = QWebEngineProfile.defaultProfile() webenginesettings._init_stylesheet(p) From 96599b96846aa108698946eb7d49ad7dc33581f0 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Fri, 17 Nov 2017 02:38:56 -0300 Subject: [PATCH 028/415] revisions set by The Compiler --- scripts/hist_importer.py | 95 ++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 48 deletions(-) mode change 100644 => 100755 scripts/hist_importer.py diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py old mode 100644 new mode 100755 index d96284879..f27b4e267 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# This file is part of qutebrowser. +# Copyright 2017 Florian Bruhin (The Compiler) +# Copyright 2014-2017 Josefson Souza +# This file is part of qutebrowser. +# # qutebrowser is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -17,60 +20,60 @@ # along with qutebrowser. If not, see . -'''Tool to import browser history data from other browsers. Although, safari -support is still on the way.''' +"""Tool to import browser history from other browsers.""" import argparse import sqlite3 import sys +import os -def parser(): +def parse(): """Parse command line arguments.""" - description = 'This program is meant to extract browser history from your'\ - 'previous browser and import them into qutebrowser.' - epilog = 'Databases:\n\tQute: Is named "history.sqlite" and can be found '\ - 'at your --basedir. In order to find where your basedir is you '\ - 'can run ":open qute:version" inside qutebrowser.'\ - '\n\tFirerox: Is named "places.sqlite", and can be found at your'\ - 'system\'s profile folder. Check this link for where it is locat'\ - 'ed: http://kb.mozillazine.org/Profile_folder'\ - '\n\tChrome: Is named "History", and can be found at the respec'\ - 'tive User Data Directory. Check this link for where it is locat'\ - 'ed: https://chromium.googlesource.com/chromium/src/+/master/'\ - 'docs/user_data_dir.md\n\n'\ - 'Example: $this_script.py -b firefox -s /Firefox/Profile/places.'\ - 'sqlite -d /qutebrowser/data/history.sqlite' - parsed = argparse.ArgumentParser( + description = ("This program is meant to extract browser history from your" + "previous browser and import them into qutebrowser.") + epilog = ("Databases:\n\tQutebrowser: Is named 'history.sqlite' and can be" + " found at your --basedir. In order to find where your basedir" + " is you can run ':open qute:version' inside qutebrowser." + "\n\tFirerox: Is named 'places.sqlite', and can be found at your" + "system\"s profile folder. Check this link for where it is locat" + "ed: http://kb.mozillazine.org/Profile_folder" + "\n\tChrome: Is named 'History', and can be found at the respec" + "tive User Data Directory. Check this link for where it is locat" + "ed: https://chromium.googlesource.com/chromium/src/+/master/" + "docs/user_data_dir.md\n\n" + "Example: hist_importer.py -b firefox -s /Firefox/Profile/" + "places.sqlite -d /qutebrowser/data/history.sqlite") + parser = argparse.ArgumentParser( description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter ) - parsed.add_argument('-b', '--browser', dest='browser', required=True, - type=str, help='Browsers: {firefox, chrome, safari}') - parsed.add_argument('-s', '--source', dest='source', required=True, - type=str, help='Source: fullpath to the sqlite data' + parser.add_argument('-b', '--browser', dest='browser', required=True, + type=str, help='Browsers: {firefox, chrome}') + parser.add_argument('-s', '--source', dest='source', required=True, + type=str, help='Source: Full path to the sqlite data' 'base file from the source browser.') - parsed.add_argument('-d', '--dest', dest='dest', required=True, type=str, - help='Destination: The fullpath to the qutebrowser ' + parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, + help='Destination: The full path to the qutebrowser ' 'sqlite database') - return parsed.parse_args() + return parser.parse_args() def open_db(data_base): """Open connection with database.""" - try: + if os.path.isfile(data_base): conn = sqlite3.connect(data_base) return conn - except Exception as any_e: - print('Error: {}'.format(any_e)) - raise('Error: There was some error trying to to connect with the [{}]' - 'database. Verify if the filepath is correct or is being used.'. - format(data_base)) + else: + raise sys.exit('\nDataBaseNotFound: There was some error trying to to' + ' connect with the [{}] database. Verify if the' + ' filepath is correct or is being used.' + .format(data_base)) def extract(source, query): - """Performs extraction of (datetime,url,title) from source.""" + """Extracts (datetime,url,title) from source database.""" try: conn = open_db(source) cursor = conn.cursor() @@ -78,18 +81,17 @@ def extract(source, query): history = cursor.fetchall() conn.close() return history - except Exception as any_e: - print('Error: {}'.format(any_e)) - print(type(source)) - raise('Error: There was some error trying to to connect with the [{}]' - 'database. Verify if the filepath is correct or is being used.'. - format(str(source))) + except sqlite3.OperationalError as op_e: + print('\nCould not perform queries into the source database: {}' + '\nBrowser version is not supported as it have a different sql' + ' schema.'.format(op_e)) def clean(history): - """Receives a list of records:(datetime,url,title). And clean all records - in place, that has a NULL/None datetime attribute. Otherwise Qutebrowser - will throw errors. Also, will add a 4rth attribute of '0' for the redirect + """Clean up records from source database. + Receives a list of records:(datetime,url,title). And clean all records + in place, that has a NULL/None datetime attribute. Otherwise qutebrowser + will throw errors. Also, will add a 4th attribute of '0' for the redirect field in history.sqlite in qutebrowser.""" nulls = [record for record in history if record[0] is None] for null_datetime in nulls: @@ -101,7 +103,8 @@ def clean(history): def insert_qb(history, dest): - """Given a list of records in history and a dest db, insert all records in + """Insert history into dest database + Given a list of records in history and a dest db, insert all records in the dest db.""" conn = open_db(dest) cursor = conn.cursor() @@ -116,7 +119,7 @@ def insert_qb(history, dest): def main(): """Main control flux of the script.""" - args = parser() + args = parse() browser = args.browser.lower() source, dest = args.source, args.dest query = { @@ -124,15 +127,11 @@ def main(): 'from moz_places', 'chrome': 'select url,title,last_visit_time/10000000 as date ' 'from urls', - 'safari': None } if browser not in query: sys.exit('Sorry, the selected browser: "{}" is not supported.'.format( browser)) else: - if browser == 'safari': - print('Sorry, currently we do not support this browser.') - sys.exit(1) history = extract(source, query[browser]) history = clean(history) insert_qb(history, dest) From 274c92a64b441297a323e2300a0f1c93de99bc1b Mon Sep 17 00:00:00 2001 From: cryzed Date: Fri, 17 Nov 2017 12:42:25 +0100 Subject: [PATCH 029/415] Add documentation additionally to the help page of qute-pass (complaint from the Arch wiki) --- misc/userscripts/qute-pass | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/misc/userscripts/qute-pass b/misc/userscripts/qute-pass index 1592d6349..4c6b1a17b 100755 --- a/misc/userscripts/qute-pass +++ b/misc/userscripts/qute-pass @@ -44,8 +44,16 @@ import sys import tldextract -argument_parser = argparse.ArgumentParser() -argument_parser.add_argument('url', nargs='?', default=os.environ['QUTE_URL']) +argument_parser = argparse.ArgumentParser(description=( + 'Insert login information using pass and a dmenu-provider (e.g. dmenu, rofi -dmenu, ...). A short ' + 'demonstration can be seen here: https://i.imgur.com/KN3XuZP.gif.'), usage=( + 'The domain of the site has to appear as a segment in the pass path, for example: "github.com/cryzed" or ' + '"websites/github.com". How the username and password are determined is freely configurable using the CLI ' + "arguments. The login information is inserted by emulating key events using qutebrowser's fake-key command in this " + 'manner: [USERNAME][PASSWORD], which is compatible with almost all login forms.'), epilog=( + "WARNING: The login details are viewable as plaintext in qutebrowser's debug log (qute://log) and might be shared " + 'if you decide to submit a crash report!')) +argument_parser.add_argument('url', nargs='?', default=os.getenv('QUTE_URL')) argument_parser.add_argument('--password-store', '-p', default=os.path.expanduser('~/.password-store'), help='Path to your pass password-store') argument_parser.add_argument('--username-pattern', '-u', default=r'.*/(.+)', @@ -71,6 +79,7 @@ stderr = functools.partial(print, file=sys.stderr) class ExitCodes(enum.IntEnum): SUCCESS = 0 + FAILURE = 1 # 1 is automatically used if Python throws an exception NO_PASS_CANDIDATES = 2 COULD_NOT_MATCH_USERNAME = 3 @@ -108,6 +117,10 @@ def dmenu(items, invocation, encoding): def main(arguments): + if not arguments.url: + argument_parser.print_help() + return ExitCodes.FAILURE + extract_result = tldextract.extract(arguments.url) # Expand potential ~ in paths, since this script won't be called from a shell that does it for us From 3131d3d3bc8acf7ba40029ffcfe59f2454d0bfbc Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Fri, 17 Nov 2017 11:48:34 -0300 Subject: [PATCH 030/415] Flake8 warnings pointed by travis. --- scripts/hist_importer.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index f27b4e267..79c723c58 100755 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -73,7 +73,15 @@ def open_db(data_base): def extract(source, query): - """Extracts (datetime,url,title) from source database.""" + """Get records from source database. + + Args: + source: File path to the source database where we want to extract the + data from. + query: The query string to be executed in order to retrieve relevant + attributes as (datetime, url, time) from the source database according + to the browser chosen. + """ try: conn = open_db(source) cursor = conn.cursor() @@ -89,10 +97,15 @@ def extract(source, query): def clean(history): """Clean up records from source database. - Receives a list of records:(datetime,url,title). And clean all records - in place, that has a NULL/None datetime attribute. Otherwise qutebrowser - will throw errors. Also, will add a 4th attribute of '0' for the redirect - field in history.sqlite in qutebrowser.""" + + Receives a list of record and sanityze them in order for them to be + properly imported to qutebrowser. Sanitation requires addiing a 4th + attribute 'redirect' which is filled with '0's, and also purging all + records that have a NULL/None datetime attribute. + + Args: + history: List of records (datetime, url, title) from source database. + """ nulls = [record for record in history if record[0] is None] for null_datetime in nulls: history.remove(null_datetime) @@ -103,9 +116,13 @@ def clean(history): def insert_qb(history, dest): - """Insert history into dest database - Given a list of records in history and a dest db, insert all records in - the dest db.""" + """Insert history into dest database. + + Args: + history: List of records. + dest: File path to the destination database, where history will be + inserted. + """ conn = open_db(dest) cursor = conn.cursor() cursor.executemany( From 0f93d532104affcab22f6915bcf47ef8b3455fdc Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 16 Nov 2017 07:38:03 -0500 Subject: [PATCH 031/415] Implement :edit-command. :edit-command opens the current command line in an editor, and updates the command line if the editor exits successfully. If --run is passed, the command is executed when the editor exits sucessfully. Resolves #2453. --- qutebrowser/mainwindow/statusbar/command.py | 19 ++++++++++++++++++- tests/end2end/features/editor.feature | 8 ++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/statusbar/command.py b/qutebrowser/mainwindow/statusbar/command.py index 9ca870605..2d45ccb22 100644 --- a/qutebrowser/mainwindow/statusbar/command.py +++ b/qutebrowser/mainwindow/statusbar/command.py @@ -24,7 +24,7 @@ from PyQt5.QtWidgets import QSizePolicy from qutebrowser.keyinput import modeman, modeparsers from qutebrowser.commands import cmdexc, cmdutils -from qutebrowser.misc import cmdhistory +from qutebrowser.misc import cmdhistory, editor from qutebrowser.misc import miscwidgets as misc from qutebrowser.utils import usertypes, log, objreg @@ -166,6 +166,23 @@ class Command(misc.MinimalLineEditMixin, misc.CommandLineEdit): modeman.leave(self._win_id, usertypes.KeyMode.command, 'cmd accept') self.got_cmd[str].emit(prefixes[text[0]] + text[1:]) + @cmdutils.register(instance='status-command', scope='window', maxsplit=0) + def edit_command(self, run=False): + """Open an editor to modify the current command. + + Args: + run: Run the command if the editor exits successfully. + """ + ed = editor.ExternalEditor(parent=self) + + def callback(text): + self.set_cmd_text(text) + if run: + self.got_cmd[str].emit(text) + + ed.editing_finished.connect(callback) + ed.edit(self.text()) + @pyqtSlot(usertypes.KeyMode) def on_mode_left(self, mode): """Clear up when command mode was left. diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index d3c634b0d..21df30b4d 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -141,3 +141,11 @@ Feature: Opening external editors And I wait for "Read back: bar" in the log And I run :click-element id qute-button Then the javascript message "text: bar" should be logged + + ## :edit-command + + Scenario: Edit a command and run it + When I run :set-cmd-text :message-info foo + And I set up a fake editor replacing "foo" by "bar" + And I run :edit-command --run + Then the message "bar" should be shown From d8887f12c03b74ec6778c04ad8c8895983c93336 Mon Sep 17 00:00:00 2001 From: cryzed Date: Fri, 17 Nov 2017 21:40:08 +0100 Subject: [PATCH 032/415] Deduplicate documentation --- misc/userscripts/qute-pass | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/misc/userscripts/qute-pass b/misc/userscripts/qute-pass index 4c6b1a17b..5bab9db93 100755 --- a/misc/userscripts/qute-pass +++ b/misc/userscripts/qute-pass @@ -17,20 +17,21 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . +""" +Insert login information using pass and a dmenu-compatible application (e.g. dmenu, rofi -dmenu, ...). A short +demonstration can be seen here: https://i.imgur.com/KN3XuZP.gif. +""" -# Insert login information using pass and a dmenu-provider (e.g. dmenu, rofi -dmenu, ...). -# A short demonstration can be seen here: https://i.imgur.com/KN3XuZP.gif. -# -# The domain of the site has to appear as a segment in the pass path, for example: "github.com/cryzed" or -# "websites/github.com". How the username and password are determined is freely configurable using the CLI arguments. -# The login information is inserted by emulating key events using qutebrowser's fake-key command in this manner: -# [USERNAME][PASSWORD], which is compatible with almost all login forms. -# -# Dependencies: tldextract (Python 3 module), pass -# For issues and feedback please use: https://github.com/cryzed/qutebrowser-userscripts. -# -# WARNING: The login details are viewable as plaintext in qutebrowser's debug log (qute://log) and might be shared if -# you decide to submit a crash report! +USAGE = """The domain of the site has to appear as a segment in the pass path, for example: "github.com/cryzed" or +"websites/github.com". How the username and password are determined is freely configurable using the CLI arguments. The +login information is inserted by emulating key events using qutebrowser's fake-key command in this manner: +[USERNAME][PASSWORD], which is compatible with almost all login forms.""" + +EPILOG = """Dependencies: tldextract (Python 3 module), pass. +For issues and feedback please use: https://github.com/cryzed/qutebrowser-userscripts. + +WARNING: The login details are viewable as plaintext in qutebrowser's debug log (qute://log) and might be shared if +you decide to submit a crash report!""" import argparse import enum @@ -44,15 +45,7 @@ import sys import tldextract -argument_parser = argparse.ArgumentParser(description=( - 'Insert login information using pass and a dmenu-provider (e.g. dmenu, rofi -dmenu, ...). A short ' - 'demonstration can be seen here: https://i.imgur.com/KN3XuZP.gif.'), usage=( - 'The domain of the site has to appear as a segment in the pass path, for example: "github.com/cryzed" or ' - '"websites/github.com". How the username and password are determined is freely configurable using the CLI ' - "arguments. The login information is inserted by emulating key events using qutebrowser's fake-key command in this " - 'manner: [USERNAME][PASSWORD], which is compatible with almost all login forms.'), epilog=( - "WARNING: The login details are viewable as plaintext in qutebrowser's debug log (qute://log) and might be shared " - 'if you decide to submit a crash report!')) +argument_parser = argparse.ArgumentParser(description=__doc__, usage=USAGE, epilog=EPILOG) argument_parser.add_argument('url', nargs='?', default=os.getenv('QUTE_URL')) argument_parser.add_argument('--password-store', '-p', default=os.path.expanduser('~/.password-store'), help='Path to your pass password-store') From 6c241f96ed626f600911cac77f357ba8eb131e0c Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Fri, 17 Nov 2017 17:49:15 -0500 Subject: [PATCH 033/415] Add test for appendChild #2723 Does some of #3295 --- .../javascript/stylesheet/test_appendchild.js | 47 +++++++++++++++++++ .../javascript/stylesheet/test_stylesheet.py | 8 ++++ 2 files changed, 55 insertions(+) create mode 100644 tests/unit/javascript/stylesheet/test_appendchild.js diff --git a/tests/unit/javascript/stylesheet/test_appendchild.js b/tests/unit/javascript/stylesheet/test_appendchild.js new file mode 100644 index 000000000..d1deadba6 --- /dev/null +++ b/tests/unit/javascript/stylesheet/test_appendchild.js @@ -0,0 +1,47 @@ +// Taken from acid3 bucket 5 +// https://github.com/w3c/web-platform-tests/blob/37cf5607a39357a0f213ab5df2e6b30499b0226f/acid/acid3/test.html#L2320 + +// test 65: bring in a couple of SVG files and some HTML files dynamically - preparation for later tests in this bucket +// NOTE FROM 2011 UPDATE: The svg.xml file still contains the SVG font, but it is no longer used +kungFuDeathGrip = document.createElement('p'); +kungFuDeathGrip.className = 'removed'; +var iframe, object; +// svg iframe +iframe = document.createElement('iframe'); +iframe.onload = function () { kungFuDeathGrip.title += '1' }; +iframe.src = "svg.xml"; +kungFuDeathGrip.appendChild(iframe); +// object iframe +object = document.createElement('object'); +object.onload = function () { kungFuDeathGrip.title += '2' }; +object.data = "svg.xml"; +kungFuDeathGrip.appendChild(object); +// xml iframe +iframe = document.createElement('iframe'); +iframe.onload = function () { kungFuDeathGrip.title += '3' }; +iframe.src = "empty.xml"; +kungFuDeathGrip.appendChild(iframe); +// html iframe +iframe = document.createElement('iframe'); +iframe.onload = function () { kungFuDeathGrip.title += '4' }; +iframe.src = "empty.html"; +kungFuDeathGrip.appendChild(iframe); +// html iframe +iframe = document.createElement('iframe'); +iframe.onload = function () { kungFuDeathGrip.title += '5' }; +iframe.src = "xhtml.1"; +kungFuDeathGrip.appendChild(iframe); +// html iframe +iframe = document.createElement('iframe'); +iframe.onload = function () { kungFuDeathGrip.title += '6' }; +iframe.src = "xhtml.2"; +kungFuDeathGrip.appendChild(iframe); +// html iframe +iframe = document.createElement('iframe'); +iframe.onload = function () { kungFuDeathGrip.title += '7' }; +iframe.src = "xhtml.3"; +kungFuDeathGrip.appendChild(iframe); +// add the lot to the document + +// Modified as we don't have a 'map' +document.getElementsByTagName('head')[0].appendChild(kungFuDeathGrip); diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index 7070cfceb..47a181d77 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -128,3 +128,11 @@ def test_set_error(stylesheet_tester): stylesheet_tester.init_stylesheet() stylesheet_tester.js.load_file('non-existent.html', force=True) stylesheet_tester.check_set(GREEN_BODY_BG) + + +def test_appendchild(stylesheet_tester): + stylesheet_tester.init_stylesheet() + stylesheet_tester.js.load('stylesheet/simple.html') + js_test_file_path = \ + '../../tests/unit/javascript/stylesheet/test_appendchild.js' + stylesheet_tester.js.run_file(js_test_file_path, {}) From 95f8c07d7fbd8e45b244e3d5d9d381fb58ea411a Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 00:31:53 +0100 Subject: [PATCH 034/415] lazy sessions --- qutebrowser/browser/qutescheme.py | 7 +++++++ qutebrowser/config/configdata.yml | 8 ++++++++ qutebrowser/misc/sessions.py | 17 ++++++++++++++++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 11dcfe004..f943fcba4 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -424,6 +424,13 @@ def qute_settings(url): confget=config.instance.get_str) return 'text/html', html +@add_handler('back') +def qute_back(url): + """Handler for qute://back. Simple page to free ram / lazy load a site, + goes back on focusing the tab.""" + + html = jinja.render('back.html', title='Suspended') + return 'text/html', html @add_handler('configdiff') def qute_configdiff(url): diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index ba8c36857..7284d98d5 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -88,6 +88,14 @@ session_default_name: If this is set to null, the session which was last loaded is saved. +session_lazy_restore: + type: + name: Bool + none_ok: true + default: false + desc: >- + Load a restored tab as soon as it takes focus. + backend: type: name: String diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 064d8c9e9..471805da9 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -347,7 +347,9 @@ class SessionManager(QObject): if 'pinned' in histentry: new_tab.data.pinned = histentry['pinned'] - active = histentry.get('active', False) + active = (histentry.get('active', False) and + (not config.val.session_lazy_restore or + histentry['url'].startswith('qute://'))) url = QUrl.fromEncoded(histentry['url'].encode('ascii')) if 'original-url' in histentry: orig_url = QUrl.fromEncoded( @@ -360,6 +362,19 @@ class SessionManager(QObject): entries.append(entry) if active: new_tab.title_changed.emit(histentry['title']) + + if config.val.session_lazy_restore and data['history']: + last = data['history'][-1] + title = last['title'] + url = 'qute://back#' + title + active = last.get('active', False) + + if not last['url'].startswith('qute://'): + entries.append(TabHistoryItem(url=QUrl.fromEncoded(url.encode('ascii')), + title=title, active=active, user_data={})) + if active: + new_tab.title_changed.emit(title) + try: new_tab.history.load_items(entries) except ValueError as e: From ade7004f8f4429affc27a8f5b03ad783ca994ca9 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 00:48:31 +0100 Subject: [PATCH 035/415] lazy sessions --- qutebrowser/browser/qutescheme.py | 2 ++ qutebrowser/misc/sessions.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index f943fcba4..3881a0d2e 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -424,6 +424,7 @@ def qute_settings(url): confget=config.instance.get_str) return 'text/html', html + @add_handler('back') def qute_back(url): """Handler for qute://back. Simple page to free ram / lazy load a site, @@ -432,6 +433,7 @@ def qute_back(url): html = jinja.render('back.html', title='Suspended') return 'text/html', html + @add_handler('configdiff') def qute_configdiff(url): """Handler for qute://configdiff.""" diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 471805da9..4784f796a 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -348,8 +348,8 @@ class SessionManager(QObject): new_tab.data.pinned = histentry['pinned'] active = (histentry.get('active', False) and - (not config.val.session_lazy_restore or - histentry['url'].startswith('qute://'))) + (not config.val.session_lazy_restore or + histentry['url'].startswith('qute://'))) url = QUrl.fromEncoded(histentry['url'].encode('ascii')) if 'original-url' in histentry: orig_url = QUrl.fromEncoded( @@ -370,8 +370,10 @@ class SessionManager(QObject): active = last.get('active', False) if not last['url'].startswith('qute://'): - entries.append(TabHistoryItem(url=QUrl.fromEncoded(url.encode('ascii')), - title=title, active=active, user_data={})) + entries.append(TabHistoryItem( + url=QUrl.fromEncoded(url.encode('ascii')), + title=title, active=active, user_data={})) + if active: new_tab.title_changed.emit(title) From 51dea053f459c7cd0977c75047ff41a2f3ca1060 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 01:00:16 +0100 Subject: [PATCH 036/415] lazy sessions --- qutebrowser/html/back.html | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 qutebrowser/html/back.html diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html new file mode 100644 index 000000000..8a21f3d80 --- /dev/null +++ b/qutebrowser/html/back.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} + +{% block script %} +window.onload = function() { + var title = 'Suspended: ' + document.location.hash.substr(1); + var node = document.getElementsByTagName('h1')[0]; + node.innerText = document.title = title; +}; +window.onfocus = function() { + window.history.back(); +}; +{% endblock %} + +{% block content %} + +

{{ title }}

+ +{% endblock %} From 633881039612e4123bebe3cf322afd5c5022021c Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Fri, 17 Nov 2017 21:42:24 -0500 Subject: [PATCH 037/415] Increase timeouts for javascript tests --- tests/unit/javascript/conftest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/javascript/conftest.py b/tests/unit/javascript/conftest.py index 7290d4d00..8490a5362 100644 --- a/tests/unit/javascript/conftest.py +++ b/tests/unit/javascript/conftest.py @@ -140,7 +140,8 @@ class JSTester: **kwargs: Passed to jinja's template.render(). """ template = self._jinja_env.get_template(path) - with self._qtbot.waitSignal(self.webview.loadFinished) as blocker: + with self._qtbot.waitSignal(self.webview.loadFinished, + timeout=2000) as blocker: self.webview.setHtml(template.render(**kwargs)) assert blocker.args == [True] @@ -161,7 +162,8 @@ class JSTester: url: The QUrl to load. force: Whether to force loading even if the file is invalid. """ - with self._qtbot.waitSignal(self.webview.loadFinished) as blocker: + with self._qtbot.waitSignal(self.webview.loadFinished, + timeout=2000) as blocker: self.webview.load(url) if not force: assert blocker.args == [True] From c4bb13431331b7c5aad5696f54998210c12a1f12 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 11:04:04 +0100 Subject: [PATCH 038/415] lazy sessions, improved version --- qutebrowser/misc/sessions.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 4784f796a..da49205df 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -323,6 +323,18 @@ class SessionManager(QObject): def _load_tab(self, new_tab, data): """Load yaml data into a newly opened tab.""" entries = [] + + if config.val.session_lazy_restore and data['history']: + last = data['history'][-1] + + if not last['url'].startswith('qute://'): + data['history'].append({ + 'title': last['title'], + 'url': 'qute://back#' + last['title'], + 'active': last.get('active', False) + }) + last['active'] = False + for histentry in data['history']: user_data = {} @@ -347,9 +359,7 @@ class SessionManager(QObject): if 'pinned' in histentry: new_tab.data.pinned = histentry['pinned'] - active = (histentry.get('active', False) and - (not config.val.session_lazy_restore or - histentry['url'].startswith('qute://'))) + active = histentry.get('active', False) url = QUrl.fromEncoded(histentry['url'].encode('ascii')) if 'original-url' in histentry: orig_url = QUrl.fromEncoded( @@ -363,20 +373,6 @@ class SessionManager(QObject): if active: new_tab.title_changed.emit(histentry['title']) - if config.val.session_lazy_restore and data['history']: - last = data['history'][-1] - title = last['title'] - url = 'qute://back#' + title - active = last.get('active', False) - - if not last['url'].startswith('qute://'): - entries.append(TabHistoryItem( - url=QUrl.fromEncoded(url.encode('ascii')), - title=title, active=active, user_data={})) - - if active: - new_tab.title_changed.emit(title) - try: new_tab.history.load_items(entries) except ValueError as e: From c150c5481a093a14c42ef1233cc54214972bc381 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 13:46:50 +0100 Subject: [PATCH 039/415] lazy sessions, dont save qute://back --- qutebrowser/misc/sessions.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index da49205df..807d4bde4 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -205,7 +205,11 @@ class SessionManager(QObject): for idx, item in enumerate(tab.history): qtutils.ensure_valid(item) item_data = self._save_tab_item(tab, idx, item) - data['history'].append(item_data) + if item_data['url'].startswith('qute://back'): + if 'active' in item_data and data['history']: + data['history'][-1]['active'] = item_data.get('active', False) + else: + data['history'].append(item_data) return data def _save_all(self, *, only_window=None, with_private=False): From 1a33c88c9607fdfd71b925699994981ce19a8827 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 13:47:57 +0100 Subject: [PATCH 040/415] lazy sessions, dont save qute://back --- qutebrowser/misc/sessions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 807d4bde4..1dc27397d 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -207,7 +207,8 @@ class SessionManager(QObject): item_data = self._save_tab_item(tab, idx, item) if item_data['url'].startswith('qute://back'): if 'active' in item_data and data['history']: - data['history'][-1]['active'] = item_data.get('active', False) + data['history'][-1]['active'] = \ + item_data.get('active', False) else: data['history'].append(item_data) return data From 2debeafe1bd5ee7dfb0a88a57957554fd03934d2 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 13:51:30 +0100 Subject: [PATCH 041/415] lazy sessions, dont save qute://back --- qutebrowser/misc/sessions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 1dc27397d..c44b2ed8b 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -206,9 +206,8 @@ class SessionManager(QObject): qtutils.ensure_valid(item) item_data = self._save_tab_item(tab, idx, item) if item_data['url'].startswith('qute://back'): - if 'active' in item_data and data['history']: - data['history'][-1]['active'] = \ - item_data.get('active', False) + if item_data.get('active', False) and data['history']: + data['history'][-1]['active'] = True else: data['history'].append(item_data) return data From cf8130bd225d4b4497f79d3ccd1d9a3226b7a6b8 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 14:12:37 +0100 Subject: [PATCH 042/415] lazy session, fix: active entry is not the end of the history --- qutebrowser/html/back.html | 14 +++++++++++--- qutebrowser/misc/sessions.py | 26 +++++++++++++++++--------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index 8a21f3d80..c28419a64 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -6,9 +6,17 @@ window.onload = function() { var node = document.getElementsByTagName('h1')[0]; node.innerText = document.title = title; }; -window.onfocus = function() { - window.history.back(); -}; +setTimeout(function() { + /* drop first focus event, to avoid problems + (allow to go easily to newer history entries) */ + var triggered = false; + window.onfocus = function() { + if (! triggered) { + triggered = true; + window.history.back(); + } + }; +}, 1000); {% endblock %} {% block content %} diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index c44b2ed8b..0e2b40c68 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -206,7 +206,9 @@ class SessionManager(QObject): qtutils.ensure_valid(item) item_data = self._save_tab_item(tab, idx, item) if item_data['url'].startswith('qute://back'): + # dont add qute://back to the session file if item_data.get('active', False) and data['history']: + # mark entry before qute://back as active data['history'][-1]['active'] = True else: data['history'].append(item_data) @@ -331,15 +333,7 @@ class SessionManager(QObject): if config.val.session_lazy_restore and data['history']: last = data['history'][-1] - if not last['url'].startswith('qute://'): - data['history'].append({ - 'title': last['title'], - 'url': 'qute://back#' + last['title'], - 'active': last.get('active', False) - }) - last['active'] = False - - for histentry in data['history']: + for i, histentry in enumerate(data['history']): user_data = {} if 'zoom' in data: @@ -363,6 +357,20 @@ class SessionManager(QObject): if 'pinned' in histentry: new_tab.data.pinned = histentry['pinned'] + if (config.val.session_lazy_restore and + histentry.get('active', False) and + not histentry['url'].startswith('qute://back')): + # remove "active" mark and insert back page marked as active + data['history'].insert( + i + 1, + { + 'title': histentry['title'], + 'url': 'qute://back#' + histentry['title'], + 'active': True + }) + histentry['active'] = False + + print(histentry) active = histentry.get('active', False) url = QUrl.fromEncoded(histentry['url'].encode('ascii')) if 'original-url' in histentry: From 13dc24f6cafe297b570b60d1ca28ae4f6a76d3c4 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 14:31:55 +0100 Subject: [PATCH 043/415] debug code removed --- qutebrowser/misc/sessions.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 0e2b40c68..c3634a64a 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -330,9 +330,6 @@ class SessionManager(QObject): """Load yaml data into a newly opened tab.""" entries = [] - if config.val.session_lazy_restore and data['history']: - last = data['history'][-1] - for i, histentry in enumerate(data['history']): user_data = {} @@ -370,7 +367,6 @@ class SessionManager(QObject): }) histentry['active'] = False - print(histentry) active = histentry.get('active', False) url = QUrl.fromEncoded(histentry['url'].encode('ascii')) if 'original-url' in histentry: From d145b304d06995e23b0796c40caa8910bae42e12 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sat, 18 Nov 2017 20:41:23 -0500 Subject: [PATCH 044/415] Remove maxsplit from edit_command. This was a copy-paste typo, no need for maxsplit as this command takes no positional args. --- qutebrowser/mainwindow/statusbar/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/statusbar/command.py b/qutebrowser/mainwindow/statusbar/command.py index 2d45ccb22..a61f3c93d 100644 --- a/qutebrowser/mainwindow/statusbar/command.py +++ b/qutebrowser/mainwindow/statusbar/command.py @@ -166,7 +166,7 @@ class Command(misc.MinimalLineEditMixin, misc.CommandLineEdit): modeman.leave(self._win_id, usertypes.KeyMode.command, 'cmd accept') self.got_cmd[str].emit(prefixes[text[0]] + text[1:]) - @cmdutils.register(instance='status-command', scope='window', maxsplit=0) + @cmdutils.register(instance='status-command', scope='window') def edit_command(self, run=False): """Open an editor to modify the current command. From 51ce5346384ae40863f497134eba671042ec3265 Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Sun, 19 Nov 2017 14:30:32 +0400 Subject: [PATCH 045/415] Document installation using FreeBSD port --- doc/install.asciidoc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/install.asciidoc b/doc/install.asciidoc index dba1410eb..cdbfa5d40 100644 --- a/doc/install.asciidoc +++ b/doc/install.asciidoc @@ -197,6 +197,21 @@ Or alternatively, use the ports system : # make install ---- +On FreeBSD +---------- + +qutebrowser is in https://www.freshports.org/www/qutebrowser/[FreeBSD ports]. + +It can be installed with: + +---- +# cd /usr/ports/www/qutebrowser +# make install clean +---- + +At present, precompiled packages are not available for this port, +and QtWebEngine backend is also not available. + On Windows ---------- From c5eab53a8750070ee36b15bd46d9b5955ecd66c1 Mon Sep 17 00:00:00 2001 From: Akhil kp Date: Sun, 19 Nov 2017 18:20:58 +0530 Subject: [PATCH 046/415] Added --file for :debug-pyeval --- qutebrowser/misc/utilcmds.py | 28 ++++++++++++++++++----- tests/end2end/data/misc/pyeval_file.py | 2 ++ tests/end2end/features/qutescheme.feature | 9 ++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 tests/end2end/data/misc/pyeval_file.py diff --git a/qutebrowser/misc/utilcmds.py b/qutebrowser/misc/utilcmds.py index 386282a25..27365011d 100644 --- a/qutebrowser/misc/utilcmds.py +++ b/qutebrowser/misc/utilcmds.py @@ -229,18 +229,34 @@ def debug_trace(expr=""): @cmdutils.register(maxsplit=0, debug=True, no_cmd_split=True) -def debug_pyeval(s, quiet=False): +def debug_pyeval(s, file=False, quiet=False): """Evaluate a python string and display the results as a web page. Args: s: The string to evaluate. + file: Interpret s as a path to file. quiet: Don't show the output in a new tab. """ - try: - r = eval(s) - out = repr(r) - except Exception: - out = traceback.format_exc() + if file: + quiet = True + path = os.path.expanduser(s) + message.info(path) + try: + with open(path, 'r', encoding='utf-8') as f: + s = f.read() + except OSError as e: + raise cmdexc.CommandError(str(e)) + try: + exec(s) + out = "No Error" + except Exception: + out = traceback.format_exc() + else: + try: + r = eval(s) + out = repr(r) + except Exception: + out = traceback.format_exc() qutescheme.pyeval_output = out if quiet: diff --git a/tests/end2end/data/misc/pyeval_file.py b/tests/end2end/data/misc/pyeval_file.py new file mode 100644 index 000000000..092ae2a54 --- /dev/null +++ b/tests/end2end/data/misc/pyeval_file.py @@ -0,0 +1,2 @@ +from qutebrowser.utils import message +message.info("Hello world") diff --git a/tests/end2end/features/qutescheme.feature b/tests/end2end/features/qutescheme.feature index e4ae20215..3b8e10822 100644 --- a/tests/end2end/features/qutescheme.feature +++ b/tests/end2end/features/qutescheme.feature @@ -171,6 +171,15 @@ Feature: Special qute:// pages And I wait until qute://pyeval/ is loaded Then the page should contain the plaintext "ZeroDivisionError" + Scenario: Running :pyveal with --file using a file that exists aspython code + When I run :debug-pyeval --file (testdata)/misc/pyeval_file.py + Then the message "Hello World" should be shown + And "pyeval output: No error" should be logged + + Scenario: Running :pyeval --file using a non existing file + When I run :debug-pyeval --file nonexestentfile + Then the error "[Errno 2] No such file or directory: 'nonexistentfile'" should be shown + Scenario: Running :pyeval with --quiet When I run :debug-pyeval --quiet 1+1 Then "pyeval output: 2" should be logged From 8fb03208e716e57122a635149ebb8003b2ce22ee Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 16 Nov 2017 21:03:27 +0100 Subject: [PATCH 047/415] Improve parsing of fatal stacktraces We now also recognize "Windows fatal exception: ..." message and refuse to send empty Windows access violation messages. --- qutebrowser/misc/crashdialog.py | 16 +++++++++++----- tests/unit/misc/test_crashdialog.py | 9 +++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/qutebrowser/misc/crashdialog.py b/qutebrowser/misc/crashdialog.py index f8b37c742..3d50e5f29 100644 --- a/qutebrowser/misc/crashdialog.py +++ b/qutebrowser/misc/crashdialog.py @@ -57,17 +57,22 @@ def parse_fatal_stacktrace(text): element being the first stacktrace frame. """ lines = [ - r'Fatal Python error: (.*)', + r'(?PFatal Python error|Windows fatal exception): (?P.*)', r' *', r'(Current )?[Tt]hread [^ ]* \(most recent call first\): *', - r' File ".*", line \d+ in (.*)', + r' File ".*", line \d+ in (?P.*)', ] m = re.match('\n'.join(lines), text) if m is None: # We got some invalid text. return ('', '') else: - return (m.group(1), m.group(3)) + msg = m.group('msg') + typ = m.group('type') + func = m.group('func') + if typ == 'Windows fatal exception': + msg = 'Windows ' + msg + return msg, func def _get_environment_vars(): @@ -474,7 +479,8 @@ class FatalCrashDialog(_CrashDialog): self._type, self._func = parse_fatal_stacktrace(self._log) def _get_error_type(self): - if self._type == 'Segmentation fault': + if self._type in ['Segmentation fault', + 'Windows access violation']: return 'segv' else: return self._type @@ -522,7 +528,7 @@ class FatalCrashDialog(_CrashDialog): """Prevent empty reports.""" if (not self._info.toPlainText().strip() and not self._contact.toPlainText().strip() and - self._type == 'Segmentation fault' and + self._get_error_type() == 'segv' and self._func == 'qt_mainloop'): msgbox.msgbox(parent=self, title='Empty crash info', text="Empty reports for fatal crashes are useless " diff --git a/tests/unit/misc/test_crashdialog.py b/tests/unit/misc/test_crashdialog.py index 89ca342ee..8bb34045c 100644 --- a/tests/unit/misc/test_crashdialog.py +++ b/tests/unit/misc/test_crashdialog.py @@ -47,6 +47,14 @@ Thread 0x00007fa135ac7700 (most recent call first): File "", line 1 in testfunc """ +WINDOWS_CRASH_TEXT = """ +Windows fatal exception: access violation +_ +Current thread 0x000014bc (most recent call first): + File "qutebrowser\mainwindow\tabbedbrowser.py", line 468 in tabopen + File "qutebrowser\browser\shared.py", line 247 in get_tab +""" + INVALID_CRASH_TEXT = """ Hello world! """ @@ -56,6 +64,7 @@ Hello world! (VALID_CRASH_TEXT, 'Segmentation fault', 'testfunc'), (VALID_CRASH_TEXT_THREAD, 'Segmentation fault', 'testfunc'), (VALID_CRASH_TEXT_EMPTY, 'Aborted', ''), + (WINDOWS_CRASH_TEXT, 'Windows access violation', 'tabopen'), (INVALID_CRASH_TEXT, '', ''), ]) def test_parse_fatal_stacktrace(text, typ, func): From 2b063f577e1f38660bd28a961cafcbe335d3da38 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 17 Nov 2017 09:09:02 +0100 Subject: [PATCH 048/415] Handle OSError in :config-write-py --- qutebrowser/config/configcommands.py | 5 ++++- tests/unit/config/test_configcommands.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/qutebrowser/config/configcommands.py b/qutebrowser/config/configcommands.py index f659b14d7..7d9adb475 100644 --- a/qutebrowser/config/configcommands.py +++ b/qutebrowser/config/configcommands.py @@ -289,4 +289,7 @@ class ConfigCommands: writer = configfiles.ConfigPyWriter(options, bindings, commented=commented) - writer.write(filename) + try: + writer.write(filename) + except OSError as e: + raise cmdexc.CommandError(str(e)) diff --git a/tests/unit/config/test_configcommands.py b/tests/unit/config/test_configcommands.py index 40a69668a..c0142b09f 100644 --- a/tests/unit/config/test_configcommands.py +++ b/tests/unit/config/test_configcommands.py @@ -413,6 +413,11 @@ class TestWritePy: lines = confpy.read_text('utf-8').splitlines() assert '# Autogenerated config.py' in lines + def test_oserror(self, commands, tmpdir): + """Test writing to a directory which does not exist.""" + with pytest.raises(cmdexc.CommandError): + commands.config_write_py(str(tmpdir / 'foo' / 'config.py')) + class TestBind: From 1a1f0fc1ee6941dbf76798140ecf9be75f157c0a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 17 Nov 2017 09:09:43 +0100 Subject: [PATCH 049/415] Update changelog --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index e78dd5d2f..24fd41b4f 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -111,6 +111,7 @@ Fixed - The `qute://gpl` page now works correctly again. - Trying to bind an empty command now doesn't crash anymore. +- Fixed crash when `:config-write-py` fails to write to the given path. v1.0.3 ------ From 8cd9cdea843edb73316e6e0dfa506e79b5337ffd Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 19 Nov 2017 14:11:20 +0100 Subject: [PATCH 050/415] Update changelog --- doc/changelog.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 24fd41b4f..584984ab7 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -43,8 +43,9 @@ Added completion item text. - New `tabs.pinned.shrink` setting to (`true` by default) to make it possible for pinned tabs and normal tabs to have the same size. -_ New `content.windowed_fullscreen` setting to show e.g. a fullscreened video in +- New `content.windowed_fullscreen` setting to show e.g. a fullscreened video in the window without fullscreening that window. +- New `:edit-command` command to edit the commandline in an editor. Changed ~~~~~~~ From 28d3771005e92a54a43a37490c601a8c16684f3b Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 19 Nov 2017 14:25:47 +0100 Subject: [PATCH 051/415] Update changelog --- doc/changelog.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 584984ab7..e202e9ece 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -72,6 +72,8 @@ Changed - The `qute://version` page now also shows the uptime of qutebrowser. - qutebrowser now prompts to create a non-existing directory when starting a download. +- Much improved user stylesheet handling which reduces flickering + and updates immediately after setting a stylesheet. Fixed ~~~~~ From 6a90cebe85a27fa398e8663e0bcfef389e3184a8 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 19 Nov 2017 14:29:22 +0100 Subject: [PATCH 052/415] Get rid of backslash --- tests/unit/javascript/stylesheet/test_stylesheet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/javascript/stylesheet/test_stylesheet.py b/tests/unit/javascript/stylesheet/test_stylesheet.py index 47a181d77..083265db7 100644 --- a/tests/unit/javascript/stylesheet/test_stylesheet.py +++ b/tests/unit/javascript/stylesheet/test_stylesheet.py @@ -133,6 +133,6 @@ def test_set_error(stylesheet_tester): def test_appendchild(stylesheet_tester): stylesheet_tester.init_stylesheet() stylesheet_tester.js.load('stylesheet/simple.html') - js_test_file_path = \ - '../../tests/unit/javascript/stylesheet/test_appendchild.js' + js_test_file_path = ('../../tests/unit/javascript/stylesheet/' + 'test_appendchild.js') stylesheet_tester.js.run_file(js_test_file_path, {}) From ba6d90aa7a07e3d1539f3003c1058186df0d2a5f Mon Sep 17 00:00:00 2001 From: akhilkpdasan Date: Sun, 19 Nov 2017 19:08:52 +0530 Subject: [PATCH 053/415] fixed docmentation for pyeval --- qutebrowser/misc/utilcmds.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qutebrowser/misc/utilcmds.py b/qutebrowser/misc/utilcmds.py index 27365011d..6f6854e2a 100644 --- a/qutebrowser/misc/utilcmds.py +++ b/qutebrowser/misc/utilcmds.py @@ -234,13 +234,12 @@ def debug_pyeval(s, file=False, quiet=False): Args: s: The string to evaluate. - file: Interpret s as a path to file. + file: Interpret s as a path to file also implies --quiete. quiet: Don't show the output in a new tab. """ if file: quiet = True path = os.path.expanduser(s) - message.info(path) try: with open(path, 'r', encoding='utf-8') as f: s = f.read() From 4644642c38dc03f93d56a161db612e5cb06f26ea Mon Sep 17 00:00:00 2001 From: akhilkpdasan Date: Sun, 19 Nov 2017 19:10:36 +0530 Subject: [PATCH 054/415] fixed test for pyeval --file --- tests/end2end/features/qutescheme.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/features/qutescheme.feature b/tests/end2end/features/qutescheme.feature index 3b8e10822..b7a2e498b 100644 --- a/tests/end2end/features/qutescheme.feature +++ b/tests/end2end/features/qutescheme.feature @@ -171,7 +171,7 @@ Feature: Special qute:// pages And I wait until qute://pyeval/ is loaded Then the page should contain the plaintext "ZeroDivisionError" - Scenario: Running :pyveal with --file using a file that exists aspython code + Scenario: Running :pyveal with --file using a file that exists as python code When I run :debug-pyeval --file (testdata)/misc/pyeval_file.py Then the message "Hello World" should be shown And "pyeval output: No error" should be logged From 9a58fe229ca5b07ea1ff8d1332843f3ec6508115 Mon Sep 17 00:00:00 2001 From: akhilkpdasan Date: Sun, 19 Nov 2017 19:32:24 +0530 Subject: [PATCH 055/415] fixed spelling error --- qutebrowser/misc/utilcmds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/misc/utilcmds.py b/qutebrowser/misc/utilcmds.py index 6f6854e2a..6f01686bf 100644 --- a/qutebrowser/misc/utilcmds.py +++ b/qutebrowser/misc/utilcmds.py @@ -234,7 +234,7 @@ def debug_pyeval(s, file=False, quiet=False): Args: s: The string to evaluate. - file: Interpret s as a path to file also implies --quiete. + file: Interpret s as a path to file, also implies --quiet. quiet: Don't show the output in a new tab. """ if file: From 21e731ebeb4c55c5ff96dd0a0df9d540eaef7d3d Mon Sep 17 00:00:00 2001 From: Akhil kp Date: Sun, 19 Nov 2017 23:49:11 +0530 Subject: [PATCH 056/415] fixed build errors(typing errors) --- tests/end2end/data/misc/pyeval_file.py | 6 +++++- tests/end2end/features/qutescheme.feature | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/end2end/data/misc/pyeval_file.py b/tests/end2end/data/misc/pyeval_file.py index 092ae2a54..27764eb20 100644 --- a/tests/end2end/data/misc/pyeval_file.py +++ b/tests/end2end/data/misc/pyeval_file.py @@ -1,2 +1,6 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +"""Simple test file for :debug-pyeval.""" + from qutebrowser.utils import message -message.info("Hello world") +message.info("Hello World") diff --git a/tests/end2end/features/qutescheme.feature b/tests/end2end/features/qutescheme.feature index b7a2e498b..8dd1a6edd 100644 --- a/tests/end2end/features/qutescheme.feature +++ b/tests/end2end/features/qutescheme.feature @@ -177,7 +177,7 @@ Feature: Special qute:// pages And "pyeval output: No error" should be logged Scenario: Running :pyeval --file using a non existing file - When I run :debug-pyeval --file nonexestentfile + When I run :debug-pyeval --file nonexistentfile Then the error "[Errno 2] No such file or directory: 'nonexistentfile'" should be shown Scenario: Running :pyeval with --quiet From c14135a6ce4fbbb1cb67a37f5894d0c2e4bf5faf Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 19 Nov 2017 19:57:44 +0100 Subject: [PATCH 057/415] Add :edit-command to docs --- doc/help/commands.asciidoc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 04377c055..29679913e 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -47,6 +47,7 @@ It is possible to run or bind multiple commands by separating them with `;;`. |<>|Open the last/[count]th download. |<>|Remove the last/[count]th download from the list. |<>|Retry the first failed/[count]th download. +|<>|Open an editor to modify the current command. |<>|Navigate to a url formed in an external editor. |<>|Enter a key mode. |<>|Send a fake keypress or key string to the website or qutebrowser. @@ -407,6 +408,15 @@ Retry the first failed/[count]th download. ==== count The index of the download to retry. +[[edit-command]] +=== edit-command +Syntax: +:edit-command [*--run*]+ + +Open an editor to modify the current command. + +==== optional arguments +* +*-r*+, +*--run*+: Run the command if the editor exits successfully. + [[edit-url]] === edit-url Syntax: +:edit-url [*--bg*] [*--tab*] [*--window*] [*--private*] [*--related*] ['url']+ From 740d629b36566074c45663de4ac0b6e06d9f1c01 Mon Sep 17 00:00:00 2001 From: akhilkpdasan Date: Mon, 20 Nov 2017 00:56:09 +0530 Subject: [PATCH 058/415] Update utilcmds.py --- qutebrowser/misc/utilcmds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/misc/utilcmds.py b/qutebrowser/misc/utilcmds.py index 6f01686bf..4054f85bb 100644 --- a/qutebrowser/misc/utilcmds.py +++ b/qutebrowser/misc/utilcmds.py @@ -247,7 +247,7 @@ def debug_pyeval(s, file=False, quiet=False): raise cmdexc.CommandError(str(e)) try: exec(s) - out = "No Error" + out = "No error" except Exception: out = traceback.format_exc() else: From 62f37df573681f54d4d698262dcd1e270a438fbc Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 19 Nov 2017 20:07:35 +0100 Subject: [PATCH 059/415] Make cursor keys go through completion if a text was entered This hopefully helps with people who try to use arrow keys for the completion, while still making the command history somewhat discoverable. --- doc/changelog.asciidoc | 4 ++++ doc/help/commands.asciidoc | 5 ++++- doc/help/configuring.asciidoc | 15 ++++++++++++--- doc/help/settings.asciidoc | 4 ++-- qutebrowser/completion/completionwidget.py | 19 ++++++++++++++++++- qutebrowser/config/configdata.yml | 4 ++-- tests/end2end/features/misc.feature | 9 +++++++++ 7 files changed, 51 insertions(+), 9 deletions(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index e202e9ece..157c8a33c 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -74,6 +74,10 @@ Changed download. - Much improved user stylesheet handling which reduces flickering and updates immediately after setting a stylesheet. +- `:completion-item-focus` now has a `--history` flag which causes it to go + through the command history when no text was entered. The default bindings for + cursor keys in the completion changed to use that, so that they can be used + again to navigate through completion items when a text was entered. Fixed ~~~~~ diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 29679913e..18e0aee92 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -1428,13 +1428,16 @@ Delete the current completion item. [[completion-item-focus]] === completion-item-focus -Syntax: +:completion-item-focus 'which'+ +Syntax: +:completion-item-focus [*--history*] 'which'+ Shift the focus of the completion menu to another item. ==== positional arguments * +'which'+: 'next', 'prev', 'next-category', or 'prev-category'. +==== optional arguments +* +*-H*+, +*--history*+: Navigate through command history if no text was typed. + [[completion-item-yank]] === completion-item-yank Syntax: +:completion-item-yank [*--sel*]+ diff --git a/doc/help/configuring.asciidoc b/doc/help/configuring.asciidoc index 347f7b49b..766e6bc7f 100644 --- a/doc/help/configuring.asciidoc +++ b/doc/help/configuring.asciidoc @@ -17,15 +17,24 @@ did in your old configuration, compared to the old defaults. Other changes in default settings: -- `` and `` in the completion now navigate through command history - instead of selecting completion items. Use ``/`` to cycle - through the completion instead. +- In v1.1.x and newer, `` and `` navigate through command history + if no text was entered yet. + With v1.0.x, they always navigate through command history instead of selecting + completion items. Use ``/`` to cycle through the completion + instead. You can get back the old behavior by doing: + ---- :bind -m command completion-item-focus prev :bind -m command completion-item-focus next ---- ++ +or always navigate through command history with ++ +---- +:bind -m command command-history-prev +:bind -m command command-history-next +---- - The default for `completion.web_history_max_items` is now set to `-1`, showing an unlimited number of items in the completion for `:open` as the new diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 64881f619..60afca0f9 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -430,13 +430,13 @@ Default: * +pass:[<Ctrl-U>]+: +pass:[rl-unix-line-discard]+ * +pass:[<Ctrl-W>]+: +pass:[rl-unix-word-rubout]+ * +pass:[<Ctrl-Y>]+: +pass:[rl-yank]+ -* +pass:[<Down>]+: +pass:[command-history-next]+ +* +pass:[<Down>]+: +pass:[completion-item-focus --history next]+ * +pass:[<Escape>]+: +pass:[leave-mode]+ * +pass:[<Return>]+: +pass:[command-accept]+ * +pass:[<Shift-Delete>]+: +pass:[completion-item-del]+ * +pass:[<Shift-Tab>]+: +pass:[completion-item-focus prev]+ * +pass:[<Tab>]+: +pass:[completion-item-focus next]+ -* +pass:[<Up>]+: +pass:[command-history-prev]+ +* +pass:[<Up>]+: +pass:[completion-item-focus --history priv]+ - +pass:[hint]+: * +pass:[<Ctrl-B>]+: +pass:[hint all tab-bg]+ diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 67864ce5f..563da166e 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -226,14 +226,31 @@ class CompletionView(QTreeView): modes=[usertypes.KeyMode.command], scope='window') @cmdutils.argument('which', choices=['next', 'prev', 'next-category', 'prev-category']) - def completion_item_focus(self, which): + @cmdutils.argument('history', flag='H') + def completion_item_focus(self, which, history=False): """Shift the focus of the completion menu to another item. Args: which: 'next', 'prev', 'next-category', or 'prev-category'. + history: Navigate through command history if no text was typed. """ if not self._active: return + + if history: + status = objreg.get('status-command', scope='window', + window=self._win_id) + if status.text() == ':' or status.history.is_browsing(): + if which == 'next': + status.command_history_next() + return + elif which == 'prev': + status.command_history_prev() + return + else: + raise cmdexc.CommandError("Can't combine --history with " + "{}!".format(which)) + selmodel = self.selectionModel() if which == 'next': diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index ba8c36857..c2d5e21bb 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -2269,8 +2269,8 @@ bindings.default: command: : command-history-prev : command-history-next - : command-history-prev - : command-history-next + : completion-item-focus --history prev + : completion-item-focus --history next : completion-item-focus prev : completion-item-focus next : completion-item-focus next-category diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index 701ff3feb..009792838 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -469,6 +469,15 @@ Feature: Various utility commands. And I run :command-accept Then the message "blah" should be shown + Scenario: Calling previous command with :completion-item-focus + When I run :set-cmd-text :message-info blah + And I run :command-accept + And I wait for "blah" in the log + And I run :set-cmd-text : + And I run :completion-item-focus prev --history + And I run :command-accept + Then the message "blah" should be shown + Scenario: Browsing through commands When I run :set-cmd-text :message-info blarg And I run :command-accept From 112800bab90afcd30100d361a6ae14da13d348ea Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 19 Nov 2017 21:04:57 +0100 Subject: [PATCH 060/415] Fix backslashes in string --- tests/unit/misc/test_crashdialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/misc/test_crashdialog.py b/tests/unit/misc/test_crashdialog.py index 8bb34045c..f8768e806 100644 --- a/tests/unit/misc/test_crashdialog.py +++ b/tests/unit/misc/test_crashdialog.py @@ -47,7 +47,7 @@ Thread 0x00007fa135ac7700 (most recent call first): File "", line 1 in testfunc """ -WINDOWS_CRASH_TEXT = """ +WINDOWS_CRASH_TEXT = r""" Windows fatal exception: access violation _ Current thread 0x000014bc (most recent call first): From 87b174b4186699c13a770d64af07323969010dc8 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 19 Nov 2017 21:07:33 +0100 Subject: [PATCH 061/415] Simplify :completion-item-focus --- qutebrowser/completion/completionwidget.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index 563da166e..f6983dc4b 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -253,16 +253,13 @@ class CompletionView(QTreeView): selmodel = self.selectionModel() - if which == 'next': - idx = self._next_idx(upwards=False) - elif which == 'prev': - idx = self._next_idx(upwards=True) - elif which == 'next-category': - idx = self._next_category_idx(upwards=False) - elif which == 'prev-category': - idx = self._next_category_idx(upwards=True) - else: # pragma: no cover - raise ValueError("Invalid 'which' value {!r}".format(which)) + indices = { + 'next': self._next_idx(upwards=False), + 'prev': self._next_idx(upwards=True), + 'next-category': self._next_category_idx(upwards=False), + 'prev-category': self._next_category_idx(upwards=True), + } + idx = indices[which] if not idx.isValid(): return From 8555b86e3bafa3c232060ebd51479311d41ca3f9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 19 Nov 2017 21:09:48 +0100 Subject: [PATCH 062/415] Add copyright notice for pyeval_file.py --- tests/end2end/data/misc/pyeval_file.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/end2end/data/misc/pyeval_file.py b/tests/end2end/data/misc/pyeval_file.py index 27764eb20..e8ede9444 100644 --- a/tests/end2end/data/misc/pyeval_file.py +++ b/tests/end2end/data/misc/pyeval_file.py @@ -1,5 +1,22 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: +# Copyright 2017 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + """Simple test file for :debug-pyeval.""" from qutebrowser.utils import message From b6bfe7c171559310ec19a3934b19988404378aa9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 19 Nov 2017 21:11:10 +0100 Subject: [PATCH 063/415] Update docs --- doc/changelog.asciidoc | 2 ++ doc/help/commands.asciidoc | 3 ++- doc/help/settings.asciidoc | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 157c8a33c..eec7ae6a7 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -78,6 +78,8 @@ Changed through the command history when no text was entered. The default bindings for cursor keys in the completion changed to use that, so that they can be used again to navigate through completion items when a text was entered. +- `:debug-pyeval` now has a `--file` arguemnt so it takes a filename instead of + a line of code. Fixed ~~~~~ diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 18e0aee92..11a8970e2 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -1786,7 +1786,7 @@ Change the log level for console logging. [[debug-pyeval]] === debug-pyeval -Syntax: +:debug-pyeval [*--quiet*] 's'+ +Syntax: +:debug-pyeval [*--file*] [*--quiet*] 's'+ Evaluate a python string and display the results as a web page. @@ -1794,6 +1794,7 @@ Evaluate a python string and display the results as a web page. * +'s'+: The string to evaluate. ==== optional arguments +* +*-f*+, +*--file*+: Interpret s as a path to file, also implies --quiet. * +*-q*+, +*--quiet*+: Don't show the output in a new tab. ==== note diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 60afca0f9..ace727686 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -436,7 +436,7 @@ Default: * +pass:[<Shift-Delete>]+: +pass:[completion-item-del]+ * +pass:[<Shift-Tab>]+: +pass:[completion-item-focus prev]+ * +pass:[<Tab>]+: +pass:[completion-item-focus next]+ -* +pass:[<Up>]+: +pass:[completion-item-focus --history priv]+ +* +pass:[<Up>]+: +pass:[completion-item-focus --history prev]+ - +pass:[hint]+: * +pass:[<Ctrl-B>]+: +pass:[hint all tab-bg]+ From 3b680d0bffc2553bbaf725f31670f4698b159adf Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 20 Nov 2017 13:54:52 +0100 Subject: [PATCH 064/415] Pass --disable-seccomp-filter-sandbox for tests with Qt 5.7.1 This is a stop-gap so I'm able to run end2end tests at least. See #3163. For unit tests, we need https://github.com/pytest-dev/pytest-qt/pull/193 first. --- tests/end2end/fixtures/quteprocess.py | 12 ++++++++---- tests/end2end/test_invocations.py | 5 ++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index a2bbd81fd..cb9265910 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -33,7 +33,7 @@ import json import yaml import pytest -from PyQt5.QtCore import pyqtSignal, QUrl +from PyQt5.QtCore import pyqtSignal, QUrl, qVersion from qutebrowser.misc import ipc from qutebrowser.utils import log, utils, javascript @@ -422,10 +422,14 @@ class QuteProc(testprocess.Process): def _default_args(self): backend = 'webengine' if self.request.config.webengine else 'webkit' - return ['--debug', '--no-err-windows', '--temp-basedir', + args = ['--debug', '--no-err-windows', '--temp-basedir', '--json-logging', '--loglevel', 'vdebug', - '--backend', backend, '--debug-flag', 'no-sql-history', - 'about:blank'] + '--backend', backend, '--debug-flag', 'no-sql-history'] + if qVersion() == '5.7.1': + # https://github.com/qutebrowser/qutebrowser/issues/3163 + args += ['--qt-flag', 'disable-seccomp-filter-sandbox'] + args.append('about:blank') + return args def path_to_url(self, path, *, port=None, https=False): """Get a URL based on a filename for the localhost webserver. diff --git a/tests/end2end/test_invocations.py b/tests/end2end/test_invocations.py index c30768ddb..dc9486142 100644 --- a/tests/end2end/test_invocations.py +++ b/tests/end2end/test_invocations.py @@ -27,7 +27,7 @@ import re import pytest -from PyQt5.QtCore import QProcess +from PyQt5.QtCore import QProcess, qVersion def _base_args(config): @@ -37,6 +37,9 @@ def _base_args(config): args += ['--backend', 'webengine'] else: args += ['--backend', 'webkit'] + if qVersion() == '5.7.1': + # https://github.com/qutebrowser/qutebrowser/issues/3163 + args += ['--qt-flag', 'disable-seccomp-filter-sandbox'] args.append('about:blank') return args From abb5c9f638e22b47e8c69c7280603be40ec08fad Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 20 Nov 2017 13:58:37 +0100 Subject: [PATCH 065/415] Ignore Qt 5.7 OpenSSL logging message --- tests/end2end/fixtures/quteprocess.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index cb9265910..5c9d9718c 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -111,6 +111,10 @@ def is_ignored_lowlevel_message(message): # file: /home/florian/#14687139 (deleted) # Error: No such file or directory return True + # Qt 5.7.1 + elif message.startswith('qt.network.ssl: QSslSocket: cannot call ' + 'unresolved function '): + return True return False From 1cc6e7190e1a5e6bcbb05670758c8d3517cbc591 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 Nov 2017 16:02:13 +0100 Subject: [PATCH 066/415] Update setuptools from 36.7.1 to 36.8.0 --- misc/requirements/requirements-pip.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pip.txt b/misc/requirements/requirements-pip.txt index 8ec830003..cbecd84d0 100644 --- a/misc/requirements/requirements-pip.txt +++ b/misc/requirements/requirements-pip.txt @@ -3,6 +3,6 @@ appdirs==1.4.3 packaging==16.8 pyparsing==2.2.0 -setuptools==36.7.1 +setuptools==36.8.0 six==1.11.0 wheel==0.30.0 From 8f0332bcf6d313009157d2fc7ab962ede6d5dc95 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 Nov 2017 16:02:14 +0100 Subject: [PATCH 067/415] Update cheroot from 5.8.3 to 5.9.1 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 42a0280a3..ff7d2d0f7 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -2,7 +2,7 @@ attrs==17.3.0 beautifulsoup4==4.6.0 -cheroot==5.8.3 +cheroot==5.9.1 click==6.7 # colorama==0.3.9 coverage==4.4.2 From b3d757d03453daf227f6b2c423eb66d53059dd95 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 Nov 2017 16:02:16 +0100 Subject: [PATCH 068/415] Update hypothesis from 3.37.0 to 3.38.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index ff7d2d0f7..8535be557 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -11,7 +11,7 @@ fields==5.0.0 Flask==0.12.2 glob2==0.6 hunter==2.0.1 -hypothesis==3.37.0 +hypothesis==3.38.0 itsdangerous==0.24 # Jinja2==2.9.6 Mako==1.0.7 From 934fb5f7d527e762519cd24158636ea403ba579f Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 Nov 2017 16:02:17 +0100 Subject: [PATCH 069/415] Update py from 1.4.34 to 1.5.2 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 8535be557..bd378f512 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -18,7 +18,7 @@ Mako==1.0.7 # MarkupSafe==1.0 parse==1.8.2 parse-type==0.4.2 -py==1.4.34 +py==1.5.2 py-cpuinfo==3.3.0 pytest==3.2.3 pytest-bdd==2.19.0 From 8dcd5708e3ac35f4c89b50bef033c10f6c0c8a28 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 Nov 2017 16:02:19 +0100 Subject: [PATCH 070/415] Update py from 1.4.34 to 1.5.2 --- misc/requirements/requirements-tox.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tox.txt b/misc/requirements/requirements-tox.txt index 76e7c1ff2..339e1de07 100644 --- a/misc/requirements/requirements-tox.txt +++ b/misc/requirements/requirements-tox.txt @@ -1,6 +1,6 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py pluggy==0.5.2 -py==1.4.34 +py==1.5.2 tox==2.9.1 virtualenv==15.1.0 From feb02769ad310261a7f018f644e85844ff112cb6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 20 Nov 2017 16:02:20 +0100 Subject: [PATCH 071/415] Update pytest from 3.2.3 to 3.2.5 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index bd378f512..4f8da07f4 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -20,7 +20,7 @@ parse==1.8.2 parse-type==0.4.2 py==1.5.2 py-cpuinfo==3.3.0 -pytest==3.2.3 +pytest==3.2.5 pytest-bdd==2.19.0 pytest-benchmark==3.1.1 pytest-catchlog==1.2.2 From 54953805808485eeb37a393ef0041425d2b28333 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Mon, 20 Nov 2017 11:39:12 -0500 Subject: [PATCH 072/415] Exit command mode after :edit-command --run. Resolves #3317, where the command prompt was left open and populated with text after running the command. --- qutebrowser/mainwindow/statusbar/command.py | 2 +- tests/end2end/features/editor.feature | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/statusbar/command.py b/qutebrowser/mainwindow/statusbar/command.py index a61f3c93d..bba2559ed 100644 --- a/qutebrowser/mainwindow/statusbar/command.py +++ b/qutebrowser/mainwindow/statusbar/command.py @@ -178,7 +178,7 @@ class Command(misc.MinimalLineEditMixin, misc.CommandLineEdit): def callback(text): self.set_cmd_text(text) if run: - self.got_cmd[str].emit(text) + self.command_accept() ed.editing_finished.connect(callback) ed.edit(self.text()) diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 21df30b4d..79f4f17d4 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -149,3 +149,4 @@ Feature: Opening external editors And I set up a fake editor replacing "foo" by "bar" And I run :edit-command --run Then the message "bar" should be shown + And "Leaving mode KeyMode.command (reason: cmd accept)" should be logged From a5d9661d7394adb6f64fb7b639841db3bdd730f0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 20 Nov 2017 19:29:31 +0100 Subject: [PATCH 073/415] Simplify is_ignored_lowlevel_message --- tests/end2end/fixtures/quteprocess.py | 94 ++++++++++++--------------- 1 file changed, 41 insertions(+), 53 deletions(-) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index 5c9d9718c..f65000a46 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -55,67 +55,55 @@ def is_ignored_qt_message(message): def is_ignored_lowlevel_message(message): """Check if we want to ignore a lowlevel process output.""" - if message.startswith('Xlib: sequence lost'): + ignored_messages = [ # https://travis-ci.org/qutebrowser/qutebrowser/jobs/157941720 # ??? - return True - elif ("_dl_allocate_tls_init: Assertion `listp->slotinfo[cnt].gen <= " - "GL(dl_tls_generation)' failed!" in message): + 'Xlib: sequence lost*', # Started appearing with Qt 5.8... # http://patchwork.sourceware.org/patch/10255/ - return True - elif message == 'getrlimit(RLIMIT_NOFILE) failed': - return True - # Travis CI containers don't have a /etc/machine-id - elif message.endswith('D-Bus library appears to be incorrectly set up; ' - 'failed to read machine uuid: Failed to open ' - '"/etc/machine-id": No such file or directory'): - return True - elif message == ('See the manual page for dbus-uuidgen to correct this ' - 'issue.'): - return True - # Travis CI macOS: - # 2017-09-11 07:32:56.191 QtWebEngineProcess[5455:28501] Couldn't set - # selectedTextBackgroundColor from default () - elif message.endswith("Couldn't set selectedTextBackgroundColor from " - "default ()"): - return True - # Mac Mini: - # <<<< VTVideoEncoderSelection >>>> - # VTSelectAndCreateVideoEncoderInstanceInternal: no video encoder found for - # 'avc1' - # - # [22:32:03.636] VTSelectAndCreateVideoEncoderInstanceInternal signalled - # err=-12908 (err) (Video encoder not available) at - # /SourceCache/CoreMedia_frameworks/CoreMedia-1562.240/Sources/ - # VideoToolbox/VTVideoEncoderSelection.c line 1245 - # - # [22:32:03.636] VTCompressionSessionCreate signalled err=-12908 (err) - # (Could not select and open encoder instance) at - # /SourceCache/CoreMedia_frameworks/CoreMedia-1562.240/Sources/ - # VideoToolbox/VTCompressionSession.c # line 946 - elif 'VTSelectAndCreateVideoEncoderInstanceInternal' in message: - return True - elif 'VTSelectAndCreateVideoEncoderInstanceInternal' in message: - return True - elif 'VTCompressionSessionCreate' in message: - return True - # During shutdown on AppVeyor: - # https://ci.appveyor.com/project/qutebrowser/qutebrowser/build/master-2089/job/or4tbct1oeqsfhfm - elif (message.startswith('QNetworkProxyFactory: factory 0x') and - message.endswith(' has returned an empty result set')): - return True - elif message == ' Error: No such file or directory': + ("*_dl_allocate_tls_init: Assertion `listp->slotinfo[cnt].gen <= " + "GL(dl_tls_generation)' failed!*"), + # ??? + 'getrlimit(RLIMIT_NOFILE) failed', + # Travis CI containers don't have a /etc/machine-id + ('*D-Bus library appears to be incorrectly set up; failed to read ' + 'machine uuid: Failed to open "/etc/machine-id": No such file or ' + 'directory'), + 'See the manual page for dbus-uuidgen to correct this issue.', + # Travis CI macOS: + # 2017-09-11 07:32:56.191 QtWebEngineProcess[5455:28501] Couldn't set + # selectedTextBackgroundColor from default () + "* Couldn't set selectedTextBackgroundColor from default ()" + # Mac Mini: + # <<<< VTVideoEncoderSelection >>>> + # VTSelectAndCreateVideoEncoderInstanceInternal: no video encoder + # found for 'avc1' + # + # [22:32:03.636] VTSelectAndCreateVideoEncoderInstanceInternal + # signalled err=-12908 (err) (Video encoder not available) at + # /SourceCache/CoreMedia_frameworks/CoreMedia-1562.240/Sources/ + # VideoToolbox/VTVideoEncoderSelection.c line 1245 + # + # [22:32:03.636] VTCompressionSessionCreate signalled err=-12908 (err) + # (Could not select and open encoder instance) at + # /SourceCache/CoreMedia_frameworks/CoreMedia-1562.240/Sources/ + # VideoToolbox/VTCompressionSession.c # line 946 + '*VTSelectAndCreateVideoEncoderInstanceInternal*', + '*VTSelectAndCreateVideoEncoderInstanceInternal*', + '*VTCompressionSessionCreate*', + # During shutdown on AppVeyor: + # https://ci.appveyor.com/project/qutebrowser/qutebrowser/build/master-2089/job/or4tbct1oeqsfhfm + 'QNetworkProxyFactory: factory 0x* has returned an empty result set', # Qt 5.10 with debug Chromium # [1016/155149.941048:WARNING:stack_trace_posix.cc(625)] Failed to open # file: /home/florian/#14687139 (deleted) # Error: No such file or directory - return True - # Qt 5.7.1 - elif message.startswith('qt.network.ssl: QSslSocket: cannot call ' - 'unresolved function '): - return True - return False + ' Error: No such file or directory', + # Qt 5.7.1 + 'qt.network.ssl: QSslSocket: cannot call unresolved function *', + ] + return any(testutils.pattern_match(pattern=pattern, value=message) + for pattern in ignored_messages) def is_ignored_chromium_message(line): From 203233c89434d35e72768b25464b670fc0f8a20a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 20 Nov 2017 20:47:40 +0100 Subject: [PATCH 074/415] Track the view correctly in the webengineview fixture This makes sure the view is closed after the test (fixing test_none on AppVeyor) and also makes sure we have a QApplication. --- tests/helpers/fixtures.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 6e17ebd71..45ec6ad09 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -315,10 +315,12 @@ def qnam(qapp): @pytest.fixture -def webengineview(): +def webengineview(qtbot): """Get a QWebEngineView if QtWebEngine is available.""" QtWebEngineWidgets = pytest.importorskip('PyQt5.QtWebEngineWidgets') - return QtWebEngineWidgets.QWebEngineView() + view = QtWebEngineWidgets.QWebEngineView() + qtbot.add_widget(view) + return view @pytest.fixture From aa40842848a60039c3978e0e7b8d0c3c4eb8f6e5 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Tue, 21 Nov 2017 00:38:51 +0100 Subject: [PATCH 075/415] lazy sessions, docstring formatted, settings renamed, javascript notice changed, insert method changed --- doc/help/commands.asciidoc | 2 +- doc/help/settings.asciidoc | 6 +++--- qutebrowser/browser/qutescheme.py | 8 +++++--- qutebrowser/config/configdata.yml | 8 +++++--- qutebrowser/html/back.html | 16 +++------------- qutebrowser/misc/sessions.py | 27 ++++++++++++++++++--------- tests/unit/misc/test_sessions.py | 2 +- 7 files changed, 36 insertions(+), 33 deletions(-) diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 04377c055..aa94de305 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -1070,7 +1070,7 @@ Syntax: +:session-save [*--current*] [*--quiet*] [*--force*] [*--only-active-win Save a session. ==== positional arguments -* +'name'+: The name of the session. If not given, the session configured in session_default_name is saved. +* +'name'+: The name of the session. If not given, the session configured in session.default_name is saved. ==== optional arguments diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 64881f619..9f694c184 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -222,7 +222,7 @@ |<>|Turn on Qt HighDPI scaling. |<>|Show a scrollbar. |<>|Enable smooth scrolling for web pages. -|<>|Name of the session to save by default. +|<>|Name of the session to save by default. |<>|Languages to use for spell checking. |<>|Hide the statusbar unless a message is shown. |<>|Padding (in pixels) for the statusbar. @@ -2554,8 +2554,8 @@ Type: <> Default: +pass:[false]+ -[[session_default_name]] -=== session_default_name +[[session.default_name]] +=== session.default_name Name of the session to save by default. If this is set to null, the session which was last loaded is saved. diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 3881a0d2e..4db7bdc84 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -427,10 +427,12 @@ def qute_settings(url): @add_handler('back') def qute_back(url): - """Handler for qute://back. Simple page to free ram / lazy load a site, - goes back on focusing the tab.""" + """Handler for qute://back. - html = jinja.render('back.html', title='Suspended') + Simple page to free ram / lazy load a site, goes back on focusing the tab. + """ + html = jinja.render('back.html', + title='Suspended: ' + url.url().split('#')[-1]) return 'text/html', html diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 7284d98d5..7f362daa4 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -79,6 +79,9 @@ new_instance_open_target_window: When `new_instance_open_target` is not set to `window`, this is ignored. session_default_name: + renamed: session.default_name + +session.default_name: type: name: SessionName none_ok: true @@ -88,13 +91,12 @@ session_default_name: If this is set to null, the session which was last loaded is saved. -session_lazy_restore: +session.lazy_restore: type: name: Bool none_ok: true default: false - desc: >- - Load a restored tab as soon as it takes focus. + desc: Load a restored tab as soon as it takes focus. backend: type: diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index c28419a64..0d06158f2 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -1,26 +1,16 @@ {% extends "base.html" %} {% block script %} -window.onload = function() { - var title = 'Suspended: ' + document.location.hash.substr(1); - var node = document.getElementsByTagName('h1')[0]; - node.innerText = document.title = title; -}; setTimeout(function() { /* drop first focus event, to avoid problems (allow to go easily to newer history entries) */ - var triggered = false; window.onfocus = function() { - if (! triggered) { - triggered = true; - window.history.back(); - } + window.onfocus = null; + window.history.back(); }; }, 1000); {% endblock %} {% block content %} - -

{{ title }}

- + {% endblock %} diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index c3634a64a..faee99c99 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -22,6 +22,8 @@ import os import os.path +from itertools import chain, dropwhile, takewhile + import sip from PyQt5.QtCore import QUrl, QObject, QPoint, QTimer from PyQt5.QtWidgets import QApplication @@ -205,8 +207,8 @@ class SessionManager(QObject): for idx, item in enumerate(tab.history): qtutils.ensure_valid(item) item_data = self._save_tab_item(tab, idx, item) - if item_data['url'].startswith('qute://back'): - # dont add qute://back to the session file + if item.url().url().startswith('qute://back'): + # don't add qute://back to the session file if item_data.get('active', False) and data['history']: # mark entry before qute://back as active data['history'][-1]['active'] = True @@ -257,7 +259,7 @@ class SessionManager(QObject): object. """ if name is default: - name = config.val.session_default_name + name = config.val.session.default_name if name is None: if self._current is not None: name = self._current @@ -329,8 +331,16 @@ class SessionManager(QObject): def _load_tab(self, new_tab, data): """Load yaml data into a newly opened tab.""" entries = [] + lazy_load = [] + # use len(data['history']) + # -> dropwhile empty if not session.lazy_session + lazy_index = len(data['history']) + gen = chain( + takewhile(lambda _: not lazy_load, enumerate(data['history'])), + enumerate(lazy_load), + dropwhile(lambda i: i[0] < lazy_index, enumerate(data['history']))) - for i, histentry in enumerate(data['history']): + for i, histentry in gen: user_data = {} if 'zoom' in data: @@ -354,13 +364,12 @@ class SessionManager(QObject): if 'pinned' in histentry: new_tab.data.pinned = histentry['pinned'] - if (config.val.session_lazy_restore and + if (config.val.session.lazy_restore and histentry.get('active', False) and not histentry['url'].startswith('qute://back')): # remove "active" mark and insert back page marked as active - data['history'].insert( - i + 1, - { + lazy_index = i + 1 + lazy_load.append({ 'title': histentry['title'], 'url': 'qute://back#' + histentry['title'], 'active': True @@ -481,7 +490,7 @@ class SessionManager(QObject): Args: name: The name of the session. If not given, the session configured - in session_default_name is saved. + in session.default_name is saved. current: Save the current session instead of the default. quiet: Don't show confirmation message. force: Force saving internal sessions (starting with an underline). diff --git a/tests/unit/misc/test_sessions.py b/tests/unit/misc/test_sessions.py index 771430d5b..b2cb8a3dd 100644 --- a/tests/unit/misc/test_sessions.py +++ b/tests/unit/misc/test_sessions.py @@ -170,7 +170,7 @@ class TestSaveAll: ]) def test_get_session_name(config_stub, sess_man, arg, config, current, expected): - config_stub.val.session_default_name = config + config_stub.val.session.default_name = config sess_man._current = current assert sess_man._get_session_name(arg) == expected From 607cd9ba6ee568849b111c2a950acb5c3ec0d9e8 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Tue, 21 Nov 2017 01:19:04 +0100 Subject: [PATCH 076/415] indent adjusted --- qutebrowser/misc/sessions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index faee99c99..7a26d161c 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -370,10 +370,10 @@ class SessionManager(QObject): # remove "active" mark and insert back page marked as active lazy_index = i + 1 lazy_load.append({ - 'title': histentry['title'], - 'url': 'qute://back#' + histentry['title'], - 'active': True - }) + 'title': histentry['title'], + 'url': 'qute://back#' + histentry['title'], + 'active': True + }) histentry['active'] = False active = histentry.get('active', False) From c3128494a1db3d16c409cbdeedba99d6c568383a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 09:19:06 +0100 Subject: [PATCH 077/415] Create CODE_OF_CONDUCT.md --- .github/CODE_OF_CONDUCT.md | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/CODE_OF_CONDUCT.md diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..549278d1e --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,46 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at mail@qutebrowser.org. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ From 273747624fb7273cfc68c82fc2771daf72fbac6f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 09:24:43 +0100 Subject: [PATCH 078/415] Update tox environment list in contributing docs --- doc/contributing.asciidoc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/doc/contributing.asciidoc b/doc/contributing.asciidoc index b39f3d902..afbb752c5 100644 --- a/doc/contributing.asciidoc +++ b/doc/contributing.asciidoc @@ -100,16 +100,10 @@ Currently, the following tox environments are available: - `py35`, `py36`: Run pytest for python 3.5/3.6 with the system-wide PyQt. - `py36-pyqt57`, ..., `py36-pyqt59`: Run pytest with the given PyQt version (`py35-*` also works). - `py36-pyqt59-cov`: Run with coverage support (other Python/PyQt versions work too). -* `flake8`: Run https://pypi.python.org/pypi/flake8[flake8] checks: - https://pypi.python.org/pypi/pyflakes[pyflakes], - https://pypi.python.org/pypi/pep8[pep8], - https://pypi.python.org/pypi/mccabe[mccabe]. +* `flake8`: Run various linting checks via https://pypi.python.org/pypi/flake8[flake8]. * `vulture`: Run https://pypi.python.org/pypi/vulture[vulture] to find unused code portions. * `pylint`: Run http://pylint.org/[pylint] static code analysis. -* `pydocstyle`: Check - https://www.python.org/dev/peps/pep-0257/[PEP257] compliance with - https://github.com/PyCQA/pydocstyle[pydocstyle]. * `pyroma`: Check packaging practices with https://pypi.python.org/pypi/pyroma/[pyroma]. * `eslint`: Run http://eslint.org/[ESLint] javascript checker. From dc00bc17746181a3042e7e345558c4a6b724ae93 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 10:35:11 +0100 Subject: [PATCH 079/415] Add an initial Makefile --- MANIFEST.in | 1 + doc/changelog.asciidoc | 3 +++ misc/Makefile | 25 +++++++++++++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 misc/Makefile diff --git a/MANIFEST.in b/MANIFEST.in index f09e9c0ef..54bb613f3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,6 +14,7 @@ include qutebrowser/git-commit-id include LICENSE doc/* README.asciidoc include misc/qutebrowser.desktop include misc/qutebrowser.appdata.xml +include misc/Makefile include requirements.txt include tox.ini include qutebrowser.py diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index eec7ae6a7..61b2df17d 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -21,6 +21,9 @@ v1.1.0 (unreleased) Added ~~~~~ +- There's now a `misc/Makefile` file in releases, which should help + distributions which package qutebrowser, as they can run something like + `make -f misc/Makefile DESTDIR="$pkgdir" install` now. - New `{current_url}` field for `window.title_format` and `tabs.title.format`. - New `colors.statusbar.passthrough.fg`/`.bg` settings. - New `completion.delay` and `completion.min_chars` settings to update the diff --git a/misc/Makefile b/misc/Makefile new file mode 100644 index 000000000..fe97eb6bf --- /dev/null +++ b/misc/Makefile @@ -0,0 +1,25 @@ +PYTHON = python3 +DESTDIR = / +ICONSIZES = 16 24 32 48 64 128 256 512 + +.PHONY: install + +doc/qutebrowser.1.html: + a2x -f manpage doc/qutebrowser.1.asciidoc + +install: doc/qutebrowser.1.html + $(PYTHON) setup.py install --root="$(DESTDIR)" --optimize=1 + install -Dm644 doc/qutebrowser.1 \ + "$(DESTDIR)/usr/share/man/man1/qutebrowser.1" + install -Dm644 misc/qutebrowser.desktop \ + "$(DESTDIR)/usr/share/applications/qutebrowser.desktop" + $(foreach i,$(ICONSIZES),install -Dm644 "icons/qutebrowser-$(i)x$(i).png" \ + "$(DESTDIR)/usr/share/icons/hicolor/$(i)x$(i)/apps/qutebrowser.png";) + install -Dm644 icons/qutebrowser.svg \ + "$(DESTDIR)/usr/share/icons/hicolor/scalable/apps/qutebrowser.svg" + install -Dm755 -t "$(DESTDIR)/usr/share/qutebrowser/userscripts/" \ + $(wildcard misc/userscripts/*) + install -Dm755 -t "$(DESTDIR)/usr/share/qutebrowser/scripts/" \ + $(filter-out scripts/__init__.py scripts/__pycache__ scripts/dev \ + scripts/testbrowser_cpp scripts/asciidoc2html.py scripts/setupcommon.py \ + scripts/link_pyqt.py,$(wildcard scripts/*)) From 54af87282550186b578f1bedf0a72de5424a6b35 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 10:45:49 +0100 Subject: [PATCH 080/415] Add missing comma --- tests/end2end/fixtures/quteprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index f65000a46..bfdefbbae 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -73,7 +73,7 @@ def is_ignored_lowlevel_message(message): # Travis CI macOS: # 2017-09-11 07:32:56.191 QtWebEngineProcess[5455:28501] Couldn't set # selectedTextBackgroundColor from default () - "* Couldn't set selectedTextBackgroundColor from default ()" + "* Couldn't set selectedTextBackgroundColor from default ()", # Mac Mini: # <<<< VTVideoEncoderSelection >>>> # VTSelectAndCreateVideoEncoderInstanceInternal: no video encoder From 56d1c5c7dd7797a6302c2b76fe592a54c5b51108 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 11:31:22 +0100 Subject: [PATCH 081/415] Stabilize command history test --- tests/end2end/features/misc.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index 009792838..dd0de1009 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -474,6 +474,7 @@ Feature: Various utility commands. And I run :command-accept And I wait for "blah" in the log And I run :set-cmd-text : + And I wait for "Entering mode KeyMode.command (reason: *)" in the log And I run :completion-item-focus prev --history And I run :command-accept Then the message "blah" should be shown From cff557d2fc194961cbe8b57d25823e64193653dd Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 13:19:11 +0100 Subject: [PATCH 082/415] Try to stabilize command test, take 2 --- tests/end2end/features/misc.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index dd0de1009..5b1f8371f 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -471,6 +471,7 @@ Feature: Various utility commands. Scenario: Calling previous command with :completion-item-focus When I run :set-cmd-text :message-info blah + And I wait for "Entering mode KeyMode.command (reason: *)" in the log And I run :command-accept And I wait for "blah" in the log And I run :set-cmd-text : From 75555fc2442eef9c3f811986319f14b22fbf739e Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 13:21:50 +0100 Subject: [PATCH 083/415] Fix warning regex --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 5da383764..edc0b2d8f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -55,7 +55,7 @@ qt_log_ignore = ^Incompatible version of OpenSSL ^QQuickWidget::invalidateRenderControl could not make context current ^libpng warning: iCCP: known incorrect sRGB profile - ^inotify_add_watch(".*") failed: "No space left on device" + ^inotify_add_watch\(".*"\) failed: "No space left on device" ^QSettings::value: Empty key passed ^Icon theme ".*" not found xfail_strict = true From 203b6c354f87b501be78f9c5230ed030b910ae09 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 14:07:49 +0100 Subject: [PATCH 084/415] Fix content.cache.size overflow with QtWebEngine While 64-bit values are allowed with QtWebKit/QNetworkDiskCache, QtWebEngine only allows 32-bit values here. With the updated sip's strict overflow checking, that means we get an exception when setting a too big value. --- doc/help/settings.asciidoc | 1 + qutebrowser/browser/webengine/webenginesettings.py | 11 +++++++++-- qutebrowser/config/configdata.yml | 6 ++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index ace727686..8ffc2407c 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -1455,6 +1455,7 @@ This setting is only available with the QtWebKit backend. [[content.cache.size]] === content.cache.size Size (in bytes) of the HTTP network cache. Null to use the default value. +With QtWebEngine, the maximum supported value is 2147483647 (~2 GB). Type: <> diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 50aa4bb18..4bf525c46 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -94,9 +94,10 @@ class DefaultProfileSetter(websettings.Base): """A setting set on the QWebEngineProfile.""" - def __init__(self, setter, default=websettings.UNSET): + def __init__(self, setter, converter=None, default=websettings.UNSET): super().__init__(default) self._setter = setter + self._converter = converter def __repr__(self): return utils.get_repr(self, setter=self._setter, constructor=True) @@ -105,7 +106,11 @@ class DefaultProfileSetter(websettings.Base): if settings is not None: raise ValueError("'settings' may not be set with " "DefaultProfileSetters!") + setter = getattr(default_profile, self._setter) + if self._converter is not None: + value = self._converter(value) + setter(value) @@ -296,7 +301,9 @@ MAPPINGS = { Attribute(QWebEngineSettings.LocalStorageEnabled), 'content.cache.size': # 0: automatically managed by QtWebEngine - DefaultProfileSetter('setHttpCacheMaximumSize', default=0), + DefaultProfileSetter('setHttpCacheMaximumSize', default=0, + converter=lambda val: + qtutils.check_overflow(val, 'int', fatal=False)), 'content.xss_auditing': Attribute(QWebEngineSettings.XSSAuditingEnabled), 'content.default_encoding': diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index c2d5e21bb..5ed18c33f 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -195,8 +195,10 @@ content.cache.size: none_ok: true minval: 0 maxval: maxint64 - desc: Size (in bytes) of the HTTP network cache. Null to use the default - value. + desc: >- + Size (in bytes) of the HTTP network cache. Null to use the default value. + + With QtWebEngine, the maximum supported value is 2147483647 (~2 GB). # Defaults from QWebSettings::QWebSettings() in # qtwebkit/Source/WebKit/qt/Api/qwebsettings.cpp From c9d42c8bea47cf344bd98f85994d693d42d89314 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 14:09:20 +0100 Subject: [PATCH 085/415] Enable strict overflow checking in tests --- tests/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 0c28ed17b..9c52f6800 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,7 @@ import os import sys import warnings +import sip import pytest import hypothesis from PyQt5.QtCore import PYQT_VERSION @@ -175,6 +176,12 @@ def pytest_configure(config): if config.webengine: import PyQt5.QtWebEngineWidgets + try: + # Added in sip 4.19.4 + sip.enableoverflowchecking(True) + except AttributeError: + pass + @pytest.fixture(scope='session', autouse=True) def check_display(request): From a451e8ac9d617fb225e8f2ed01ae7bd3153f3ffe Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 14:09:30 +0100 Subject: [PATCH 086/415] Add a cache_tmpdir fixture --- tests/helpers/fixtures.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 45ec6ad09..5cbf01aad 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -458,6 +458,18 @@ def runtime_tmpdir(monkeypatch, tmpdir): return runtimedir +@pytest.fixture +def cache_tmpdir(monkeypatch, tmpdir): + """Set tmpdir/cache as the cachedir. + + Use this to avoid creating a 'real' cache dir (~/.cache/qute_test). + """ + cachedir = tmpdir / 'cache' + cachedir.ensure(dir=True) + monkeypatch.setattr(standarddir, 'cache', lambda: str(cachedir)) + return cachedir + + @pytest.fixture def redirect_webengine_data(data_tmpdir, monkeypatch): """Set XDG_DATA_HOME and HOME to a temp location. From 6c9f496edf1e4d5fcab08adf6424e715c48cb72f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 14:09:38 +0100 Subject: [PATCH 087/415] Add a test for setting content.cache.size to a big value --- .../webengine/test_webenginesettings.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/unit/browser/webengine/test_webenginesettings.py diff --git a/tests/unit/browser/webengine/test_webenginesettings.py b/tests/unit/browser/webengine/test_webenginesettings.py new file mode 100644 index 000000000..7a3e0971b --- /dev/null +++ b/tests/unit/browser/webengine/test_webenginesettings.py @@ -0,0 +1,36 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +import pytest + +from qutebrowser.browser.webengine import webenginesettings + + +@pytest.fixture(autouse=True) +def init_profiles(qapp, config_stub, cache_tmpdir, data_tmpdir): + webenginesettings._init_profiles() + + +def test_big_cache_size(config_stub): + """Make sure a too big cache size is handled correctly.""" + config_stub.val.content.cache.size = 2 ** 63 - 1 + webenginesettings._update_settings('content.cache.size') + + size = webenginesettings.default_profile.httpCacheMaximumSize() + assert size == 2 ** 31 - 1 From 2f12233cb79f7f673fd4cdc51e8191cdef118f24 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 14:10:48 +0100 Subject: [PATCH 088/415] Update changelog --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 61b2df17d..aaa6a45f7 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -146,6 +146,7 @@ Fixed window - Unbinding a default keybinding twice now doesn't bind it again - Completions are now sorted correctly again when filtered +- Fix crash when setting content.cache.size to a big value (> 2 GB) v1.0.2 ------ From 8440303d826001ffcf49e29c50006db7a07c7253 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 14:53:31 +0100 Subject: [PATCH 089/415] Stabilize cleaning up open tabs For some reason, about:blank isn't properly loaded sometimes. But this should always work. --- tests/end2end/features/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/features/conftest.py b/tests/end2end/features/conftest.py index 7f5b4e2a6..90d26b468 100644 --- a/tests/end2end/features/conftest.py +++ b/tests/end2end/features/conftest.py @@ -162,7 +162,7 @@ def clean_open_tabs(quteproc): quteproc.send_cmd(':window-only') quteproc.send_cmd(':tab-only --force') quteproc.send_cmd(':tab-close --force') - quteproc.wait_for_load_finished_url('about:blank') + quteproc.wait_for(message="Changing title for idx 0 to 'about:blank'") @bdd.given('pdfjs is available') From 9744a3d0bcadaac752ac924743a0eb20dcd5fa02 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 17:58:32 +0100 Subject: [PATCH 090/415] Skip test_webenginesettings without QtWebEngine --- tests/unit/browser/webengine/test_webenginesettings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/browser/webengine/test_webenginesettings.py b/tests/unit/browser/webengine/test_webenginesettings.py index 7a3e0971b..6c58e29af 100644 --- a/tests/unit/browser/webengine/test_webenginesettings.py +++ b/tests/unit/browser/webengine/test_webenginesettings.py @@ -19,6 +19,8 @@ import pytest +pytest.importorskip('PyQt5.QtWebEngineWidgets') + from qutebrowser.browser.webengine import webenginesettings From b684e50cdf9f47503c64733edcfab4f12033f8fa Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 21 Nov 2017 18:14:17 +0100 Subject: [PATCH 091/415] Stabilize view-source test --- tests/end2end/features/tabs.feature | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/end2end/features/tabs.feature b/tests/end2end/features/tabs.feature index 77487d33d..f28ae62ec 100644 --- a/tests/end2end/features/tabs.feature +++ b/tests/end2end/features/tabs.feature @@ -628,7 +628,8 @@ Feature: Tab management @qtwebkit_skip @qt<5.9 Scenario: Cloning a tab with a view-source URL - When I open view-source:http://localhost:(port) + When I open / + And I open view-source:http://localhost:(port) And I run :tab-clone Then the error "Can't serialize special URL!" should be shown From 7532db83c44374740ccaf0a8cd268f998843f2fc Mon Sep 17 00:00:00 2001 From: Vladimir Shulyak Date: Tue, 21 Nov 2017 18:57:41 +0000 Subject: [PATCH 092/415] Add option to persist current mode on tab change --- qutebrowser/config/configdata.yml | 5 +++++ qutebrowser/mainwindow/tabbedbrowser.py | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 5ed18c33f..56a0801c4 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1235,6 +1235,11 @@ tabs.padding: type: Padding desc: Padding (in pixels) around text for tabs. +tabs.persist_mode_on_change: + default: false + type: Bool + desc: When switching tabs, retain the current mode. + tabs.position: default: top type: Position diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index 2338d5dc9..9b26c4028 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -652,9 +652,10 @@ class TabbedBrowser(tabwidget.TabWidget): log.modes.debug("Current tab changed, focusing {!r}".format(tab)) tab.setFocus() - for mode in [usertypes.KeyMode.hint, usertypes.KeyMode.insert, - usertypes.KeyMode.caret, usertypes.KeyMode.passthrough]: - modeman.leave(self._win_id, mode, 'tab changed', maybe=True) + if not config.val.tabs.persist_mode_on_change: + for mode in [usertypes.KeyMode.hint, usertypes.KeyMode.insert, + usertypes.KeyMode.caret, usertypes.KeyMode.passthrough]: + modeman.leave(self._win_id, mode, 'tab changed', maybe=True) if self._now_focused is not None: objreg.register('last-focused-tab', self._now_focused, update=True, scope='window', window=self._win_id) From 358c888760de8bfa39ec5cdc330388eea46dd9dd Mon Sep 17 00:00:00 2001 From: Vladimir Shulyak Date: Tue, 21 Nov 2017 22:35:19 +0000 Subject: [PATCH 093/415] Fix long line --- qutebrowser/mainwindow/tabbedbrowser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index 9b26c4028..9480adc5f 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -654,7 +654,8 @@ class TabbedBrowser(tabwidget.TabWidget): tab.setFocus() if not config.val.tabs.persist_mode_on_change: for mode in [usertypes.KeyMode.hint, usertypes.KeyMode.insert, - usertypes.KeyMode.caret, usertypes.KeyMode.passthrough]: + usertypes.KeyMode.caret, + usertypes.KeyMode.passthrough]: modeman.leave(self._win_id, mode, 'tab changed', maybe=True) if self._now_focused is not None: objreg.register('last-focused-tab', self._now_focused, update=True, From e2d5a443cc746885f6fb3512470298b168904e29 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Tue, 21 Nov 2017 23:57:06 +0100 Subject: [PATCH 094/415] lazy sessions --- qutebrowser/browser/qutescheme.py | 2 +- qutebrowser/config/configdata.yml | 4 +--- qutebrowser/html/back.html | 2 +- qutebrowser/misc/sessions.py | 15 ++++++++------- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 4db7bdc84..f12ef529a 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -432,7 +432,7 @@ def qute_back(url): Simple page to free ram / lazy load a site, goes back on focusing the tab. """ html = jinja.render('back.html', - title='Suspended: ' + url.url().split('#')[-1]) + title='Suspended: ' + url.fragment()) return 'text/html', html diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 7f362daa4..39c0fa6b1 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -92,9 +92,7 @@ session.default_name: If this is set to null, the session which was last loaded is saved. session.lazy_restore: - type: - name: Bool - none_ok: true + type: Bool default: false desc: Load a restored tab as soon as it takes focus. diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index 0d06158f2..83f5cb74e 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -12,5 +12,5 @@ setTimeout(function() { {% endblock %} {% block content %} - + {% endblock %} diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 7a26d161c..64d170dab 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -21,8 +21,7 @@ import os import os.path - -from itertools import chain, dropwhile, takewhile +import itertools import sip from PyQt5.QtCore import QUrl, QObject, QPoint, QTimer @@ -207,7 +206,7 @@ class SessionManager(QObject): for idx, item in enumerate(tab.history): qtutils.ensure_valid(item) item_data = self._save_tab_item(tab, idx, item) - if item.url().url().startswith('qute://back'): + if item.url().scheme() == 'qute' and item.url().host() == 'back': # don't add qute://back to the session file if item_data.get('active', False) and data['history']: # mark entry before qute://back as active @@ -335,10 +334,12 @@ class SessionManager(QObject): # use len(data['history']) # -> dropwhile empty if not session.lazy_session lazy_index = len(data['history']) - gen = chain( - takewhile(lambda _: not lazy_load, enumerate(data['history'])), - enumerate(lazy_load), - dropwhile(lambda i: i[0] < lazy_index, enumerate(data['history']))) + gen = itertools.chain( + itertools.takewhile(lambda _: not lazy_load, + enumerate(data['history'])), + enumerate(lazy_load), + itertools.dropwhile(lambda i: i[0] < lazy_index, + enumerate(data['history']))) for i, histentry in gen: user_data = {} From c4291a8ed57623cc75cea7b2d45c27b27aa6a2d9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 07:52:03 +0100 Subject: [PATCH 095/415] Update GitHub contributing docs --- .github/CONTRIBUTING.asciidoc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.asciidoc b/.github/CONTRIBUTING.asciidoc index 6c64f296a..6449c6323 100644 --- a/.github/CONTRIBUTING.asciidoc +++ b/.github/CONTRIBUTING.asciidoc @@ -4,6 +4,9 @@ - Either run the testsuite locally, or keep an eye on Travis CI / AppVeyor after pushing changes. -See the full contribution docs for details: +- If you are stuck somewhere or have questions, + https://github.com/qutebrowser/qutebrowser#getting-help[please ask]! + +See the full contribution documentation for details and other useful hints: include::../doc/contributing.asciidoc[] From 8acfe501fe736268e44fb17f14cdb5d18adc9d81 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 07:56:31 +0100 Subject: [PATCH 096/415] Revert "Stabilize cleaning up open tabs" We're going to fix this in a more generic way. This reverts commit 8440303d826001ffcf49e29c50006db7a07c7253. --- tests/end2end/features/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/features/conftest.py b/tests/end2end/features/conftest.py index 90d26b468..7f5b4e2a6 100644 --- a/tests/end2end/features/conftest.py +++ b/tests/end2end/features/conftest.py @@ -162,7 +162,7 @@ def clean_open_tabs(quteproc): quteproc.send_cmd(':window-only') quteproc.send_cmd(':tab-only --force') quteproc.send_cmd(':tab-close --force') - quteproc.wait_for(message="Changing title for idx 0 to 'about:blank'") + quteproc.wait_for_load_finished_url('about:blank') @bdd.given('pdfjs is available') From aa417019ae6ac8f5ae9ea172f90c26a33efc9755 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 07:58:55 +0100 Subject: [PATCH 097/415] Try stabilizing loading about:blank in end2end tests --- tests/end2end/fixtures/quteprocess.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index bfdefbbae..7b9f69d57 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -676,20 +676,26 @@ class QuteProc(testprocess.Process): else: timeout = 5000 - # We really need the same representation that the webview uses in its - # __repr__ qurl = QUrl(url) if not qurl.isValid(): raise ValueError("Invalid URL {}: {}".format(url, qurl.errorString())) - url = utils.elide(qurl.toDisplayString(QUrl.EncodeUnicode), 100) - assert url - pattern = re.compile( - r"(load status for : LoadStatus\.{load_status}|fetch: " - r"PyQt5\.QtCore\.QUrl\('{url}'\) -> .*)".format( - load_status=re.escape(load_status), url=re.escape(url))) + if qurl == QUrl('about:blank'): + # For some reason, we don't get a LoadStatus.success for about:blank + # sometimes. + pattern = "Changing title for idx * to 'about:blank'" + else: + # We really need the same representation that the webview uses in its + # __repr__ + url = utils.elide(qurl.toDisplayString(QUrl.EncodeUnicode), 100) + assert url + + pattern = re.compile( + r"(load status for : LoadStatus\.{load_status}|fetch: " + r"PyQt5\.QtCore\.QUrl\('{url}'\) -> .*)".format( + load_status=re.escape(load_status), url=re.escape(url))) try: self.wait_for(message=pattern, timeout=timeout) From fd93e7ba6c040f9ad56c0614f5e1e5074d01c46e Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 08:02:38 +0100 Subject: [PATCH 098/415] Update docs --- doc/changelog.asciidoc | 2 ++ doc/help/settings.asciidoc | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index aaa6a45f7..4931c1199 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -49,6 +49,8 @@ Added - New `content.windowed_fullscreen` setting to show e.g. a fullscreened video in the window without fullscreening that window. - New `:edit-command` command to edit the commandline in an editor. +- New `tabs.persist_mode_on_change` setting to keep the current mode when + switching tabs. Changed ~~~~~~~ diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 8ffc2407c..d1fcf2822 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -239,6 +239,7 @@ |<>|Position of new tabs opened from another tab. |<>|Position of new tabs which aren't opened from another tab. |<>|Padding (in pixels) around text for tabs. +|<>|When switching tabs, retain the current mode. |<>|Shrink pinned tabs down to their contents. |<>|Position of the tab bar. |<>|Which tab to select when the focused tab is removed. @@ -2799,6 +2800,14 @@ Default: - +pass:[right]+: +pass:[5]+ - +pass:[top]+: +pass:[0]+ +[[tabs.persist_mode_on_change]] +=== tabs.persist_mode_on_change +When switching tabs, retain the current mode. + +Type: <> + +Default: +pass:[false]+ + [[tabs.pinned.shrink]] === tabs.pinned.shrink Shrink pinned tabs down to their contents. From e5cabb6d23910cae68e2030d756b3285868fe607 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 08:39:33 +0100 Subject: [PATCH 099/415] Match QtWebKit error message for qute://help/img test --- tests/end2end/features/qutescheme.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/features/qutescheme.feature b/tests/end2end/features/qutescheme.feature index 8dd1a6edd..51db7f767 100644 --- a/tests/end2end/features/qutescheme.feature +++ b/tests/end2end/features/qutescheme.feature @@ -76,7 +76,7 @@ Feature: Special qute:// pages Scenario: Opening a link with qute://help/img/ When the documentation is up to date And I open qute://help/img/ without waiting - Then "OSError while handling qute://* URL" should be logged + Then "*Error while * qute://*" should be logged And "* url='qute://help/img'* LoadStatus.error" should be logged # :history From 401a37bf4bf9aeb3dad7de13bfa1e10c5af71d99 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 08:43:47 +0100 Subject: [PATCH 100/415] Stabilize :spawn with userscript tests --- tests/end2end/features/spawn.feature | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/end2end/features/spawn.feature b/tests/end2end/features/spawn.feature index e2b5fdd5b..2a1ea0039 100644 --- a/tests/end2end/features/spawn.feature +++ b/tests/end2end/features/spawn.feature @@ -40,21 +40,21 @@ Feature: :spawn @posix Scenario: Running :spawn with userscript - When I open about:blank + When I open data/hello.txt And I run :spawn -u (testdata)/userscripts/open_current_url - And I wait until about:blank is loaded + And I wait until data/hello.txt is loaded Then the following tabs should be open: - - about:blank - - about:blank (active) + - data/hello.txt + - data/hello.txt (active) @windows Scenario: Running :spawn with userscript on Windows - When I open about:blank + When I open data/hello.txt And I run :spawn -u (testdata)/userscripts/open_current_url.bat - And I wait until about:blank is loaded + And I wait until data/hello.txt is loaded Then the following tabs should be open: - - about:blank - - about:blank (active) + - data/hello.txt + - data/hello.txt (active) @posix Scenario: Running :spawn with userscript that expects the stdin getting closed From 2581be051fbc25ebd416de6474c48726acf58e02 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 08:46:15 +0100 Subject: [PATCH 101/415] Always leave hint/caret mode when switching tabs See #3323 --- doc/help/settings.asciidoc | 4 ++-- qutebrowser/config/configdata.yml | 2 +- qutebrowser/mainwindow/tabbedbrowser.py | 11 +++++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index d1fcf2822..f2a2ba5da 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -239,7 +239,7 @@ |<>|Position of new tabs opened from another tab. |<>|Position of new tabs which aren't opened from another tab. |<>|Padding (in pixels) around text for tabs. -|<>|When switching tabs, retain the current mode. +|<>|When switching tabs, stay in insert/passthrough mode. |<>|Shrink pinned tabs down to their contents. |<>|Position of the tab bar. |<>|Which tab to select when the focused tab is removed. @@ -2802,7 +2802,7 @@ Default: [[tabs.persist_mode_on_change]] === tabs.persist_mode_on_change -When switching tabs, retain the current mode. +When switching tabs, stay in insert/passthrough mode. Type: <> diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 56a0801c4..822cef76b 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1238,7 +1238,7 @@ tabs.padding: tabs.persist_mode_on_change: default: false type: Bool - desc: When switching tabs, retain the current mode. + desc: When switching tabs, stay in insert/passthrough mode. tabs.position: default: top diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py index 9480adc5f..c78c0eced 100644 --- a/qutebrowser/mainwindow/tabbedbrowser.py +++ b/qutebrowser/mainwindow/tabbedbrowser.py @@ -652,11 +652,14 @@ class TabbedBrowser(tabwidget.TabWidget): log.modes.debug("Current tab changed, focusing {!r}".format(tab)) tab.setFocus() + + modes_to_leave = [usertypes.KeyMode.hint, usertypes.KeyMode.caret] if not config.val.tabs.persist_mode_on_change: - for mode in [usertypes.KeyMode.hint, usertypes.KeyMode.insert, - usertypes.KeyMode.caret, - usertypes.KeyMode.passthrough]: - modeman.leave(self._win_id, mode, 'tab changed', maybe=True) + modes_to_leave += [usertypes.KeyMode.insert, + usertypes.KeyMode.passthrough] + for mode in modes_to_leave: + modeman.leave(self._win_id, mode, 'tab changed', maybe=True) + if self._now_focused is not None: objreg.register('last-focused-tab', self._now_focused, update=True, scope='window', window=self._win_id) From 3b3acba34e99812532972e6fda6b392238f0d7f7 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 09:11:17 +0100 Subject: [PATCH 102/415] Edit description of tabs.persist_mode_on_change --- doc/help/settings.asciidoc | 4 ++-- qutebrowser/config/configdata.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index f2a2ba5da..3f6a8a016 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -239,7 +239,7 @@ |<>|Position of new tabs opened from another tab. |<>|Position of new tabs which aren't opened from another tab. |<>|Padding (in pixels) around text for tabs. -|<>|When switching tabs, stay in insert/passthrough mode. +|<>|Stay in insert/passthrough mode when switching tabs. |<>|Shrink pinned tabs down to their contents. |<>|Position of the tab bar. |<>|Which tab to select when the focused tab is removed. @@ -2802,7 +2802,7 @@ Default: [[tabs.persist_mode_on_change]] === tabs.persist_mode_on_change -When switching tabs, stay in insert/passthrough mode. +Stay in insert/passthrough mode when switching tabs. Type: <> diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 822cef76b..08c854ed3 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1238,7 +1238,7 @@ tabs.padding: tabs.persist_mode_on_change: default: false type: Bool - desc: When switching tabs, stay in insert/passthrough mode. + desc: Stay in insert/passthrough mode when switching tabs. tabs.position: default: top From 12f4940ef36c5f032ed2b75f4a40886b36438bd2 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 09:35:16 +0100 Subject: [PATCH 103/415] Make :jseval use a fixed path with relative paths --- doc/changelog.asciidoc | 2 ++ doc/help/commands.asciidoc | 5 ++++- qutebrowser/browser/commands.py | 8 +++++++- tests/end2end/features/misc.feature | 4 ++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 4931c1199..78edb1ddf 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -85,6 +85,8 @@ Changed again to navigate through completion items when a text was entered. - `:debug-pyeval` now has a `--file` arguemnt so it takes a filename instead of a line of code. +- `:jseval --file` now searches relative paths in a js/ subdir in qutebrowser's + data dir, e.g. `~/.local/share/qutebrowser/js`. Fixed ~~~~~ diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 11a8970e2..95caf24f1 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -639,7 +639,10 @@ Evaluate a JavaScript string. * +'js-code'+: The string/file to evaluate. ==== optional arguments -* +*-f*+, +*--file*+: Interpret js-code as a path to a file. +* +*-f*+, +*--file*+: Interpret js-code as a path to a file. If the path is relative, the file is searched in a js/ subdir + in qutebrowser's data dir, e.g. + `~/.local/share/qutebrowser/js`. + * +*-q*+, +*--quiet*+: Don't show resulting JS object. * +*-w*+, +*--world*+: Ignored on QtWebKit. On QtWebEngine, a world ID or name to run the snippet in. diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index e007de76d..8c80cd590 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -39,7 +39,7 @@ from qutebrowser.browser import (urlmarks, browsertab, inspector, navigate, webelem, downloads) from qutebrowser.keyinput import modeman from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils, - objreg, utils, debug) + objreg, utils, debug, standarddir) from qutebrowser.utils.usertypes import KeyMode from qutebrowser.misc import editor, guiprocess from qutebrowser.completion.models import urlmodel, miscmodels @@ -2029,6 +2029,9 @@ class CommandDispatcher: Args: js_code: The string/file to evaluate. file: Interpret js-code as a path to a file. + If the path is relative, the file is searched in a js/ subdir + in qutebrowser's data dir, e.g. + `~/.local/share/qutebrowser/js`. quiet: Don't show resulting JS object. world: Ignored on QtWebKit. On QtWebEngine, a world ID or name to run the snippet in. @@ -2058,6 +2061,9 @@ class CommandDispatcher: if file: path = os.path.expanduser(js_code) + if not os.path.isabs(path): + path = os.path.join(standarddir.data(), 'js') + try: with open(path, 'r', encoding='utf-8') as f: js_code = f.read() diff --git a/tests/end2end/features/misc.feature b/tests/end2end/features/misc.feature index 5b1f8371f..70505c1f8 100644 --- a/tests/end2end/features/misc.feature +++ b/tests/end2end/features/misc.feature @@ -118,8 +118,8 @@ Feature: Various utility commands. And "No output or error" should be logged Scenario: :jseval --file using a file that doesn't exist as js-code - When I run :jseval --file nonexistentfile - Then the error "[Errno 2] No such file or directory: 'nonexistentfile'" should be shown + When I run :jseval --file /nonexistentfile + Then the error "[Errno 2] No such file or directory: '/nonexistentfile'" should be shown And "No output or error" should not be logged # :debug-webaction From 221b81ae9023a28bf4f54d0e53632b3737e80d8e Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 09:35:36 +0100 Subject: [PATCH 104/415] Fix typo in changelog --- doc/changelog.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 78edb1ddf..cb7a7ba37 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -83,7 +83,7 @@ Changed through the command history when no text was entered. The default bindings for cursor keys in the completion changed to use that, so that they can be used again to navigate through completion items when a text was entered. -- `:debug-pyeval` now has a `--file` arguemnt so it takes a filename instead of +- `:debug-pyeval` now has a `--file` argument so it takes a filename instead of a line of code. - `:jseval --file` now searches relative paths in a js/ subdir in qutebrowser's data dir, e.g. `~/.local/share/qutebrowser/js`. From cbf5fc01fa7396d183fa897a1360409935ac2389 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 10:13:35 +0100 Subject: [PATCH 105/415] Avoid more about:blank loads --- tests/end2end/features/history.feature | 3 +-- tests/end2end/features/urlmarks.feature | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/end2end/features/history.feature b/tests/end2end/features/history.feature index aa126c4f7..00bd20403 100644 --- a/tests/end2end/features/history.feature +++ b/tests/end2end/features/history.feature @@ -5,8 +5,7 @@ Feature: Page history Make sure the global page history is saved correctly. Background: - Given I open about:blank - And I run :history-clear --force + Given I run :history-clear --force Scenario: Simple history saving When I open data/numbers/1.txt diff --git a/tests/end2end/features/urlmarks.feature b/tests/end2end/features/urlmarks.feature index c171cbcd3..836af5c4f 100644 --- a/tests/end2end/features/urlmarks.feature +++ b/tests/end2end/features/urlmarks.feature @@ -89,9 +89,9 @@ Feature: quickmarks and bookmarks Then the bookmark file should not contain "http://localhost:*/data/numbers/5.txt " Scenario: Deleting the current page's bookmark if it doesn't exist - When I open about:blank + When I open data/hello.txt And I run :bookmark-del - Then the error "Bookmark 'about:blank' not found!" should be shown + Then the error "Bookmark 'http://localhost:(port)/data/hello.txt' not found!" should be shown Scenario: Deleting the current page's bookmark When I open data/numbers/6.txt @@ -212,9 +212,9 @@ Feature: quickmarks and bookmarks Then the quickmark file should not contain "eighteen http://localhost:*/data/numbers/18.txt " Scenario: Deleting the current page's quickmark if it has none - When I open about:blank + When I open data/hello.txt And I run :quickmark-del - Then the error "Quickmark for 'about:blank' not found!" should be shown + Then the error "Quickmark for 'http://localhost:(port)/data/hello.txt' not found!" should be shown Scenario: Deleting the current page's quickmark When I open data/numbers/19.txt From 487951cd31d2be173bf0e93d38ba71cd9955136e Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 10:57:48 +0100 Subject: [PATCH 106/415] Fix joining :jseval path --- qutebrowser/browser/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 8c80cd590..dc91a321e 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -2062,7 +2062,7 @@ class CommandDispatcher: if file: path = os.path.expanduser(js_code) if not os.path.isabs(path): - path = os.path.join(standarddir.data(), 'js') + path = os.path.join(standarddir.data(), 'js', path) try: with open(path, 'r', encoding='utf-8') as f: From 72d466d236b7b045fc69e71db4a1a49a87daa773 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 11:34:10 +0100 Subject: [PATCH 107/415] Break long lines --- tests/end2end/fixtures/quteprocess.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index 7b9f69d57..af19b8583 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -682,12 +682,12 @@ class QuteProc(testprocess.Process): qurl.errorString())) if qurl == QUrl('about:blank'): - # For some reason, we don't get a LoadStatus.success for about:blank - # sometimes. + # For some reason, we don't get a LoadStatus.success for + # about:blank sometimes. pattern = "Changing title for idx * to 'about:blank'" else: - # We really need the same representation that the webview uses in its - # __repr__ + # We really need the same representation that the webview uses in + # its __repr__ url = utils.elide(qurl.toDisplayString(QUrl.EncodeUnicode), 100) assert url From 5d8c9577a723a7e80773b13cd7b0de376b7dc9a2 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 11:53:49 +0100 Subject: [PATCH 108/415] Improve hypothesis example generation for configtypes.Dict --- tests/unit/config/test_configtypes.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unit/config/test_configtypes.py b/tests/unit/config/test_configtypes.py index 36988a0a7..1a4360479 100644 --- a/tests/unit/config/test_configtypes.py +++ b/tests/unit/config/test_configtypes.py @@ -1616,19 +1616,19 @@ class TestDict: else: d.to_py(val) - @hypothesis.given(val=strategies.dictionaries(strategies.text(min_size=1), - strategies.booleans())) + @hypothesis.given(val=strategies.dictionaries( + strategies.text(min_size=1, alphabet=strategies.characters( + # No control characters, surrogates, or codepoints encoded as + # surrogate + blacklist_categories=['Cc', 'Cs'], max_codepoint=0xFFFF)), + strategies.booleans())) def test_hypothesis(self, klass, val): d = klass(keytype=configtypes.String(), valtype=configtypes.Bool(), none_ok=True) - try: - converted = d.to_py(val) - expected = converted if converted else None - assert d.from_str(d.to_str(converted)) == expected - except configexc.ValidationError: - # Invalid unicode in the string, etc... - hypothesis.assume(False) + converted = d.to_py(val) + expected = converted if converted else None + assert d.from_str(d.to_str(converted)) == expected @hypothesis.given(val=strategies.dictionaries(strategies.text(min_size=1), strategies.booleans())) From 750d2c490ff8baa8e76af7686df7fec6ae572594 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 14:56:25 +0100 Subject: [PATCH 109/415] Fix :completion-item-focus --history with / or ? If we have no completion (like when searching), we can just always go through history with up/down. --- qutebrowser/completion/completionwidget.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index f6983dc4b..b72a14d3c 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -234,13 +234,10 @@ class CompletionView(QTreeView): which: 'next', 'prev', 'next-category', or 'prev-category'. history: Navigate through command history if no text was typed. """ - if not self._active: - return - if history: status = objreg.get('status-command', scope='window', window=self._win_id) - if status.text() == ':' or status.history.is_browsing(): + if status.text() == ':' or status.history.is_browsing() or not self._active: if which == 'next': status.command_history_next() return @@ -251,8 +248,10 @@ class CompletionView(QTreeView): raise cmdexc.CommandError("Can't combine --history with " "{}!".format(which)) - selmodel = self.selectionModel() + if not self._active: + return + selmodel = self.selectionModel() indices = { 'next': self._next_idx(upwards=False), 'prev': self._next_idx(upwards=True), From b31360b6e3cdd1bda27f63c6dcc42934ce27db70 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 22 Nov 2017 17:19:21 +0100 Subject: [PATCH 110/415] Fix line break --- qutebrowser/completion/completionwidget.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/completion/completionwidget.py b/qutebrowser/completion/completionwidget.py index b72a14d3c..29471597f 100644 --- a/qutebrowser/completion/completionwidget.py +++ b/qutebrowser/completion/completionwidget.py @@ -237,7 +237,8 @@ class CompletionView(QTreeView): if history: status = objreg.get('status-command', scope='window', window=self._win_id) - if status.text() == ':' or status.history.is_browsing() or not self._active: + if (status.text() == ':' or status.history.is_browsing() or + not self._active): if which == 'next': status.command_history_next() return From 8eab4028209b99284c413f0a1c89af9aeca8d011 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 23 Nov 2017 08:07:11 -0500 Subject: [PATCH 111/415] Abort :edit-command on invalid input. Show an error message if the user edits the command such that it is missing a start character (:, /, or ?). Previously, this would cause the browser to crash. Resolves #3326. --- qutebrowser/mainwindow/statusbar/command.py | 6 +++++- tests/end2end/features/editor.feature | 13 +++++++++++++ tests/end2end/features/test_editor_bdd.py | 5 +++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/statusbar/command.py b/qutebrowser/mainwindow/statusbar/command.py index bba2559ed..affb1a23b 100644 --- a/qutebrowser/mainwindow/statusbar/command.py +++ b/qutebrowser/mainwindow/statusbar/command.py @@ -26,7 +26,7 @@ from qutebrowser.keyinput import modeman, modeparsers from qutebrowser.commands import cmdexc, cmdutils from qutebrowser.misc import cmdhistory, editor from qutebrowser.misc import miscwidgets as misc -from qutebrowser.utils import usertypes, log, objreg +from qutebrowser.utils import usertypes, log, objreg, message class Command(misc.MinimalLineEditMixin, misc.CommandLineEdit): @@ -176,6 +176,10 @@ class Command(misc.MinimalLineEditMixin, misc.CommandLineEdit): ed = editor.ExternalEditor(parent=self) def callback(text): + if not text or text[0] not in modeparsers.STARTCHARS: + message.error('command must start with one of {}' + .format(modeparsers.STARTCHARS)) + return self.set_cmd_text(text) if run: self.command_accept() diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 79f4f17d4..93b61df30 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -150,3 +150,16 @@ Feature: Opening external editors And I run :edit-command --run Then the message "bar" should be shown And "Leaving mode KeyMode.command (reason: cmd accept)" should be logged + + Scenario: Edit a command and omit the start char + When I set up a fake editor returning "message-info foo" + And I run :edit-command + Then the error "command must start with one of :/?" should be shown + And "Leaving mode KeyMode.command *" should not be logged + + Scenario: Edit a command to be empty + When I run :set-cmd-text : + When I set up a fake editor returning empty text + And I run :edit-command + Then the error "command must start with one of :/?" should be shown + And "Leaving mode KeyMode.command *" should not be logged diff --git a/tests/end2end/features/test_editor_bdd.py b/tests/end2end/features/test_editor_bdd.py index eb937e0f2..3133bea40 100644 --- a/tests/end2end/features/test_editor_bdd.py +++ b/tests/end2end/features/test_editor_bdd.py @@ -58,3 +58,8 @@ def set_up_editor(quteproc, server, tmpdir, text): """.format(text=text))) editor = json.dumps([sys.executable, str(script), '{}']) quteproc.set_setting('editor.command', editor) + +@bdd.when(bdd.parsers.parse('I set up a fake editor returning empty text')) +def set_up_editor_empty(quteproc, server, tmpdir): + """Set up editor.command to a small python script inserting empty text.""" + set_up_editor(quteproc, server, tmpdir, "") From 739cfc03ba9a06d42cf0053e87e40abaf8a3b500 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Thu, 23 Nov 2017 23:14:21 -0500 Subject: [PATCH 112/415] Fix undercounting short hints --- qutebrowser/browser/hints.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/hints.py b/qutebrowser/browser/hints.py index 8d8a5ae68..b68d00058 100644 --- a/qutebrowser/browser/hints.py +++ b/qutebrowser/browser/hints.py @@ -446,8 +446,17 @@ class HintManager(QObject): # Short hints are the number of hints we can possibly show which are # (needed - 1) digits in length. if needed > min_chars: - short_count = math.floor((len(chars) ** needed - len(elems)) / - len(chars)) + total_space = len(chars) ** needed + # Calculate short_count naively, by finding the avaiable space and + # dividing by the number of spots we would loose by adding a + # short element + short_count = math.floor((total_space - len(elems)) / + (len(chars))) + # Check if we double counted above to warrant another short_count + # https://github.com/qutebrowser/qutebrowser/issues/3242 + if total_space - (short_count * len(chars) + + (len(elems) - short_count)) >= len(chars) - 1: + short_count += 1 else: short_count = 0 From 28d7c5e204da79f3a1b929311ae6e00559d34f72 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 24 Nov 2017 07:25:10 +0100 Subject: [PATCH 113/415] Fix lint --- qutebrowser/mainwindow/statusbar/command.py | 1 + tests/end2end/features/test_editor_bdd.py | 1 + 2 files changed, 2 insertions(+) diff --git a/qutebrowser/mainwindow/statusbar/command.py b/qutebrowser/mainwindow/statusbar/command.py index affb1a23b..af2dc3dc9 100644 --- a/qutebrowser/mainwindow/statusbar/command.py +++ b/qutebrowser/mainwindow/statusbar/command.py @@ -176,6 +176,7 @@ class Command(misc.MinimalLineEditMixin, misc.CommandLineEdit): ed = editor.ExternalEditor(parent=self) def callback(text): + """Set the commandline to the edited text.""" if not text or text[0] not in modeparsers.STARTCHARS: message.error('command must start with one of {}' .format(modeparsers.STARTCHARS)) diff --git a/tests/end2end/features/test_editor_bdd.py b/tests/end2end/features/test_editor_bdd.py index 3133bea40..f66b94f77 100644 --- a/tests/end2end/features/test_editor_bdd.py +++ b/tests/end2end/features/test_editor_bdd.py @@ -59,6 +59,7 @@ def set_up_editor(quteproc, server, tmpdir, text): editor = json.dumps([sys.executable, str(script), '{}']) quteproc.set_setting('editor.command', editor) + @bdd.when(bdd.parsers.parse('I set up a fake editor returning empty text')) def set_up_editor_empty(quteproc, server, tmpdir): """Set up editor.command to a small python script inserting empty text.""" From 0afd6b23c9fdf62323a5b3b29d83bf9ead622041 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 24 Nov 2017 09:23:35 +0100 Subject: [PATCH 114/415] Add the option name to the backend error message --- qutebrowser/config/config.py | 2 +- qutebrowser/config/configexc.py | 6 +++--- tests/unit/config/test_configcommands.py | 4 ++-- tests/unit/config/test_configexc.py | 5 +++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index 802695b8e..ff09e2a5b 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -257,7 +257,7 @@ class Config(QObject): """Set the given option to the given value.""" if not isinstance(objects.backend, objects.NoBackend): if objects.backend not in opt.backends: - raise configexc.BackendError(objects.backend) + raise configexc.BackendError(opt.name, objects.backend) opt.typ.to_py(value) # for validation self._values[opt.name] = opt.typ.from_obj(value) diff --git a/qutebrowser/config/configexc.py b/qutebrowser/config/configexc.py index 1199a9864..28b269dd5 100644 --- a/qutebrowser/config/configexc.py +++ b/qutebrowser/config/configexc.py @@ -35,9 +35,9 @@ class BackendError(Error): """Raised when this setting is unavailable with the current backend.""" - def __init__(self, backend): - super().__init__("This setting is not available with the {} " - "backend!".format(backend.name)) + def __init__(self, name, backend): + super().__init__("The {} setting is not available with the {} " + "backend!".format(name, backend.name)) class ValidationError(Error): diff --git a/tests/unit/config/test_configcommands.py b/tests/unit/config/test_configcommands.py index c0142b09f..7137f50db 100644 --- a/tests/unit/config/test_configcommands.py +++ b/tests/unit/config/test_configcommands.py @@ -130,8 +130,8 @@ class TestSet: def test_set_wrong_backend(self, commands, monkeypatch): monkeypatch.setattr(objects, 'backend', usertypes.Backend.QtWebEngine) with pytest.raises(cmdexc.CommandError, - match="This setting is not available with the " - "QtWebEngine backend!"): + match="The content.cookies.accept setting is not " + "available with the QtWebEngine backend!"): commands.set(0, 'content.cookies.accept', 'all') @pytest.mark.parametrize('option', ['?', 'url.auto_search']) diff --git a/tests/unit/config/test_configexc.py b/tests/unit/config/test_configexc.py index 03248731b..8fd99a9c7 100644 --- a/tests/unit/config/test_configexc.py +++ b/tests/unit/config/test_configexc.py @@ -49,8 +49,9 @@ def test_no_option_error_clash(): def test_backend_error(): - e = configexc.BackendError(usertypes.Backend.QtWebKit) - assert str(e) == "This setting is not available with the QtWebKit backend!" + e = configexc.BackendError('foo', usertypes.Backend.QtWebKit) + expected = "The foo setting is not available with the QtWebKit backend!" + assert str(e) == expected def test_desc_with_text(): From 6cc78aeaeeb295b429d39668bfad74c076b96b55 Mon Sep 17 00:00:00 2001 From: Lucas Hoffmann Date: Fri, 24 Nov 2017 13:32:13 +0100 Subject: [PATCH 115/415] Register qutebrowser as a handler for qute:// links These links mostly occur within qutebrowser's documentation. But the are also written to the auto generated config file. Clicking them in any application that consults the desktop database (or uses xdg-open) will thus open them in qutebrowser correctly. --- misc/qutebrowser.desktop | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/qutebrowser.desktop b/misc/qutebrowser.desktop index e505774a8..96cbda392 100644 --- a/misc/qutebrowser.desktop +++ b/misc/qutebrowser.desktop @@ -7,5 +7,5 @@ Categories=Network;WebBrowser; Exec=qutebrowser %u Terminal=false StartupNotify=false -MimeType=text/html;text/xml;application/xhtml+xml;application/xml;application/rdf+xml;image/gif;image/jpeg;image/png;x-scheme-handler/http;x-scheme-handler/https; +MimeType=text/html;text/xml;application/xhtml+xml;application/xml;application/rdf+xml;image/gif;image/jpeg;image/png;x-scheme-handler/http;x-scheme-handler/https;x-scheme-handler/qute; Keywords=Browser From 15c7ede91679d922b3f4a20360b16ab3369001e4 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 24 Nov 2017 14:15:35 +0100 Subject: [PATCH 116/415] Update to PyQt 5.9.2 --- README.asciidoc | 2 +- misc/requirements/requirements-pyqt.txt | 4 ++-- misc/requirements/requirements-pyqt.txt-raw | 4 +--- tox.ini | 8 ++++---- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/README.asciidoc b/README.asciidoc index 48603d2be..a87165366 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -109,7 +109,7 @@ The following software and libraries are required to run qutebrowser: link:https://github.com/annulen/webkit/wiki[updated fork] (5.212) is supported * http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.7.0 or newer - (5.9 recommended) for Python 3 + (5.9.2 recommended) for Python 3 * https://pypi.python.org/pypi/setuptools/[pkg_resources/setuptools] * http://fdik.org/pyPEG/[pyPEG2] * http://jinja.pocoo.org/[jinja2] diff --git a/misc/requirements/requirements-pyqt.txt b/misc/requirements/requirements-pyqt.txt index 0ced8d869..5a08f2f73 100644 --- a/misc/requirements/requirements-pyqt.txt +++ b/misc/requirements/requirements-pyqt.txt @@ -1,4 +1,4 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -PyQt5==5.9 # rq.filter: != 5.9.1 -sip==4.19.5 +PyQt5==5.9.2 +sip==4.19.6 diff --git a/misc/requirements/requirements-pyqt.txt-raw b/misc/requirements/requirements-pyqt.txt-raw index bca0092dc..37a69c45a 100644 --- a/misc/requirements/requirements-pyqt.txt-raw +++ b/misc/requirements/requirements-pyqt.txt-raw @@ -1,3 +1 @@ -#@ filter: PyQt5 != 5.9.1 - -PyQt5==5.9 \ No newline at end of file +PyQt5 \ No newline at end of file diff --git a/tox.ini b/tox.ini index 662496f88..c0395a621 100644 --- a/tox.ini +++ b/tox.ini @@ -86,7 +86,7 @@ setenv = passenv = {[testenv]passenv} deps = {[testenv]deps} - PyQt5==5.9 + PyQt5==5.9.2 commands = {envpython} -bb -m pytest {posargs:tests} [testenv:py36-pyqt59] @@ -97,7 +97,7 @@ setenv = passenv = {[testenv]passenv} deps = {[testenv]deps} - PyQt5==5.9 + PyQt5==5.9.2 commands = {envpython} -bb -m pytest {posargs:tests} # test envs with coverage @@ -110,7 +110,7 @@ setenv = passenv = {[testenv]passenv} deps = {[testenv]deps} - PyQt5==5.9 + PyQt5==5.9.2 commands = {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests} {envpython} scripts/dev/check_coverage.py {posargs} @@ -123,7 +123,7 @@ setenv = passenv = {[testenv]passenv} deps = {[testenv]deps} - PyQt5==5.9 + PyQt5==5.9.2 commands = {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests} {envpython} scripts/dev/check_coverage.py {posargs} From 06fc52321e0a860ed8b1d50b8beb038880d88770 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 24 Nov 2017 14:17:31 +0100 Subject: [PATCH 117/415] Ignore new Qt 5.9.3 error message --- tests/end2end/fixtures/quteprocess.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index af19b8583..27720e43e 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -139,6 +139,11 @@ def is_ignored_chromium_message(line): # channel message 'Invalid node channel message', + # Qt 5.9.3 + # [30217:30229:1124/141512.682110:ERROR:cert_verify_proc_openssl.cc(212)] + # X509 Verification error self signed certificate : 18 : 0 : 4 + 'X509 Verification error self signed certificate : 18 : 0 : 4', + # Not reproducible anymore? 'Running without the SUID sandbox! *', From 765a22189c3effa6482e87089739a21d438e261f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8B=BC=E8=80=B3?= Date: Fri, 24 Nov 2017 14:52:33 +0100 Subject: [PATCH 118/415] check if qutebrowser process is running Sometimes when qutebrowser crashes, it leaves the IPC socket file behind. In those cases this script still tried to use it, failed, and didn't open your URL at all. --- scripts/open_url_in_instance.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/open_url_in_instance.sh b/scripts/open_url_in_instance.sh index 119c3aa4f..e9c6db2a1 100755 --- a/scripts/open_url_in_instance.sh +++ b/scripts/open_url_in_instance.sh @@ -6,13 +6,14 @@ _url="$1" _qb_version='0.10.1' _proto_version=1 _ipc_socket="${XDG_RUNTIME_DIR}/qutebrowser/ipc-$(echo -n "$USER" | md5sum | cut -d' ' -f1)" +_qute_bin="/usr/bin/qutebrowser" -if [[ -e "${_ipc_socket}" ]]; then +if [[ -e "${_ipc_socket}" ]] && [[ `pgrep -f $_qute_bin` ]]; then exec printf '{"args": ["%s"], "target_arg": null, "version": "%s", "protocol_version": %d, "cwd": "%s"}\n' \ "${_url}" \ "${_qb_version}" \ "${_proto_version}" \ "${PWD}" | socat - UNIX-CONNECT:"${_ipc_socket}" else - exec /usr/bin/qutebrowser --backend webengine "$@" + exec $_qute_bin --backend webengine "$@" fi From 03a9cbdfb446ba0f5d94fc6def889183fc03184b Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 24 Nov 2017 16:03:21 +0100 Subject: [PATCH 119/415] Break long lines (cherry picked from commit 18a45bbd5b0492785cfc6e9b541d816ed8ec5c54) --- tests/end2end/fixtures/quteprocess.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index 27720e43e..27c347ca4 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -140,7 +140,8 @@ def is_ignored_chromium_message(line): 'Invalid node channel message', # Qt 5.9.3 - # [30217:30229:1124/141512.682110:ERROR:cert_verify_proc_openssl.cc(212)] + # [30217:30229:1124/141512.682110:ERROR: + # cert_verify_proc_openssl.cc(212)] # X509 Verification error self signed certificate : 18 : 0 : 4 'X509 Verification error self signed certificate : 18 : 0 : 4', From 9df149fe8fc0822fcdc6a718e3542782aca22fe6 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Fri, 24 Nov 2017 17:15:26 +0100 Subject: [PATCH 120/415] urlencode fix --- qutebrowser/browser/qutescheme.py | 6 ++++-- qutebrowser/misc/sessions.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index f12ef529a..9771f6db1 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -29,6 +29,7 @@ import os import time import textwrap import mimetypes +import urllib import pkg_resources from PyQt5.QtCore import QUrlQuery, QUrl @@ -431,8 +432,9 @@ def qute_back(url): Simple page to free ram / lazy load a site, goes back on focusing the tab. """ - html = jinja.render('back.html', - title='Suspended: ' + url.fragment()) + html = jinja.render( + 'back.html', + title='Suspended: ' + urllib.parse.unquote(url.fragment())) return 'text/html', html diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 64d170dab..1b0106f1b 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -22,6 +22,7 @@ import os import os.path import itertools +import urllib import sip from PyQt5.QtCore import QUrl, QObject, QPoint, QTimer @@ -372,7 +373,9 @@ class SessionManager(QObject): lazy_index = i + 1 lazy_load.append({ 'title': histentry['title'], - 'url': 'qute://back#' + histentry['title'], + 'url': + 'qute://back#' + + urllib.parse.quote(histentry['title']), 'active': True }) histentry['active'] = False From c9af36909f054fff59cfdb0d2a58a441b6149d54 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Fri, 24 Nov 2017 13:21:21 -0500 Subject: [PATCH 121/415] Add tests for hint scattering --- tests/end2end/test_hints_html.py | 47 ++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/end2end/test_hints_html.py b/tests/end2end/test_hints_html.py index abc106505..7b84bd635 100644 --- a/tests/end2end/test_hints_html.py +++ b/tests/end2end/test_hints_html.py @@ -22,12 +22,17 @@ import os import os.path import textwrap +import string +import functools +import operator import attr import yaml import pytest import bs4 +import qutebrowser.browser.hints + def collect_tests(): basedir = os.path.dirname(__file__) @@ -146,3 +151,45 @@ def test_word_hints_issue1393(quteproc, tmpdir): quteproc.wait_for(message='hints: *', category='hints') quteproc.send_cmd(':follow-hint {}'.format(hint)) quteproc.wait_for_load_finished('data/{}'.format(target)) + + +@pytest.mark.parametrize('min_len', [0, 3]) +@pytest.mark.parametrize('num_chars', [9]) +@pytest.mark.parametrize('num_elements', range(1, 26)) +def test_scattered_hints_count(win_registry, mode_manager, min_len, + num_chars, num_elements): + """Test scattered hints function.""" + # pylint: disable=W0141 + manager = qutebrowser.browser.hints.HintManager(0, 0) + chars = string.ascii_lowercase[:num_chars] + + hints = manager._hint_scattered(min_len, chars, + list(range(num_elements))) + + # Check if hints are unique + assert len(hints) == len(set(hints)) + + # Check if any hints are shorter than min_len + assert not any([x for x in hints if len(x) < min_len]) + + # Check we don't have more than 2 link lengths + # Eg: 'a' 'bc' and 'def' cannot be in the same hint string + hint_lens = set(map(len, hints)) + assert len(hint_lens) <= 2 + + if len(hint_lens) == 2: + # Check if hint_lens are more than 1 apart + # Eg: 'abc' and 'd' cannot be in the same hint sequence, but + # 'ab' and 'c' can + assert abs(functools.reduce(operator.sub, hint_lens)) <= 1 + + longest_hints = filter(lambda x: len(x) == max(hint_lens), hints) + + if min_len < max(hint_lens) - 1: + # Check if we have any unique prefixes. For example, 'la' + # alone, with no other 'l' + count_map = {} + for x in longest_hints: + prefix = x[:-1] + count_map[prefix] = count_map.get(prefix, 0) + 1 + assert not any(filter(lambda x: x == 1, count_map.values())) From e8db59a9efb8ea0322352579c18f76cfc773d117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8B=BC=E8=80=B3?= Date: Sat, 25 Nov 2017 11:09:57 +0100 Subject: [PATCH 122/415] Use socat exit status to determine if the socket is usable Instead of checking, if *any* qutebrowser process is running (which may or may not have an IPC socket where we expect it), simply launch a new instance *if socat fails*. Which it does, if: * the socket file doesn't exist (qutebrowser simply not running), or * the socket isn't connectable (qutebrowser crashed, left orphaned socket) Also put new instances into background, so the script behaves a bit more consistently. (Else it *sometimes* blocks and *sometimes doesn't*, when run.) --- scripts/open_url_in_instance.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/open_url_in_instance.sh b/scripts/open_url_in_instance.sh index e9c6db2a1..85cd919d4 100755 --- a/scripts/open_url_in_instance.sh +++ b/scripts/open_url_in_instance.sh @@ -8,12 +8,8 @@ _proto_version=1 _ipc_socket="${XDG_RUNTIME_DIR}/qutebrowser/ipc-$(echo -n "$USER" | md5sum | cut -d' ' -f1)" _qute_bin="/usr/bin/qutebrowser" -if [[ -e "${_ipc_socket}" ]] && [[ `pgrep -f $_qute_bin` ]]; then - exec printf '{"args": ["%s"], "target_arg": null, "version": "%s", "protocol_version": %d, "cwd": "%s"}\n' \ - "${_url}" \ - "${_qb_version}" \ - "${_proto_version}" \ - "${PWD}" | socat - UNIX-CONNECT:"${_ipc_socket}" -else - exec $_qute_bin --backend webengine "$@" -fi +exec printf '{"args": ["%s"], "target_arg": null, "version": "%s", "protocol_version": %d, "cwd": "%s"}\n' \ + "${_url}" \ + "${_qb_version}" \ + "${_proto_version}" \ + "${PWD}" | socat - UNIX-CONNECT:"${_ipc_socket}" 2>/dev/null || $_qute_bin --backend webengine "$@" & From 5ed801911595cc7b485db66196b5fcb9017ec4d9 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Fri, 24 Nov 2017 09:25:01 -0600 Subject: [PATCH 123/415] update flake8 and flake8-deprecated Updated requirements and adjusted the configuration in `.flake8`; other files have been modified where the lack of per-file auto-ignore caused problems, where putty's `# flake8: disable=` syntax could be replaced with a simpler `noqa`, or where pylint directives already suppressed the same error. --- .flake8 | 18 ++++++------------ misc/requirements/requirements-flake8.txt | 8 +++----- misc/requirements/requirements-flake8.txt-raw | 13 +++---------- qutebrowser/browser/qutescheme.py | 2 +- .../browser/webkit/network/networkmanager.py | 2 +- .../browser/webkit/network/networkreply.py | 2 +- qutebrowser/commands/cmdutils.py | 4 ++-- qutebrowser/commands/command.py | 2 +- qutebrowser/config/config.py | 2 +- qutebrowser/keyinput/modeman.py | 2 +- qutebrowser/mainwindow/statusbar/bar.py | 2 +- qutebrowser/misc/crashsignal.py | 2 +- qutebrowser/misc/split.py | 2 +- qutebrowser/utils/debug.py | 2 +- qutebrowser/utils/log.py | 4 ++-- qutebrowser/utils/utils.py | 2 +- tests/conftest.py | 2 +- tests/helpers/fixtures.py | 2 +- tests/helpers/stubs.py | 2 +- tests/unit/commands/test_cmdutils.py | 2 +- tests/unit/config/test_config.py | 2 +- tests/unit/utils/test_debug.py | 4 ++-- 22 files changed, 34 insertions(+), 49 deletions(-) diff --git a/.flake8 b/.flake8 index 1d33859fc..e29e7037c 100644 --- a/.flake8 +++ b/.flake8 @@ -33,18 +33,12 @@ ignore = D102,D103,D104,D105,D209,D211,D402,D403 min-version = 3.4.0 max-complexity = 12 -putty-auto-ignore = True -putty-ignore = - /# pylint: disable=invalid-name/ : +N801,N806 - /# pragma: no mccabe/ : +C901 - tests/*/test_*.py : +D100,D101,D401 - tests/conftest.py : +F403 - tests/unit/browser/test_history.py : +N806 - tests/helpers/fixtures.py : +N806 - tests/unit/browser/webkit/http/test_content_disposition.py : +D400 - scripts/dev/ci/appveyor_install.py : +FI53 - # FIXME:conf - tests/unit/completion/test_models.py : +F821 +per-file-ignores = + tests/*/test_*.py : D100,D101,D401 + tests/unit/browser/test_history.py : N806 + tests/helpers/fixtures.py : N806 + tests/unit/browser/webkit/http/test_content_disposition.py : D400 + scripts/dev/ci/appveyor_install.py : FI53 copyright-check = True copyright-regexp = # Copyright [\d-]+ .* copyright-min-file-size = 110 diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index 053d06ec4..ff33c19e1 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -1,23 +1,21 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -flake8==2.6.2 # rq.filter: < 3.0.0 +flake8==3.5.0 flake8-copyright==0.2.0 flake8-debugger==3.0.0 -flake8-deprecated==1.2.1 # rq.filter: < 1.3 +flake8-deprecated==1.3 flake8-docstrings==1.0.3 # rq.filter: < 1.1.0 flake8-future-import==0.4.3 flake8-mock==0.3 flake8-pep3101==1.0 # rq.filter: < 1.1 +flake8-per-file-ignores==0.4 flake8-polyfill==1.0.1 -flake8-putty==0.4.0 flake8-string-format==0.2.3 flake8-tidy-imports==1.1.0 flake8-tuple==0.2.13 mccabe==0.6.1 -packaging==16.8 pep8-naming==0.4.1 pycodestyle==2.3.1 pydocstyle==1.1.1 # rq.filter: < 2.0.0 pyflakes==1.6.0 -pyparsing==2.2.0 six==1.11.0 diff --git a/misc/requirements/requirements-flake8.txt-raw b/misc/requirements/requirements-flake8.txt-raw index 7f0b5153a..2baced0d0 100644 --- a/misc/requirements/requirements-flake8.txt-raw +++ b/misc/requirements/requirements-flake8.txt-raw @@ -1,12 +1,12 @@ -flake8<3.0.0 +flake8 flake8-copyright flake8-debugger -flake8-deprecated<1.3 +flake8-deprecated flake8-docstrings<1.1.0 flake8-future-import flake8-mock flake8-pep3101<1.1 -flake8-putty +flake8-per-file-ignores flake8-string-format flake8-tidy-imports flake8-tuple @@ -14,14 +14,7 @@ pep8-naming pydocstyle<2.0.0 pyflakes -# Pinned to 2.0.0 otherwise -pycodestyle==2.3.1 -# Pinned to 0.5.3 otherwise -mccabe==0.6.1 - # Waiting until flake8-putty updated -#@ filter: flake8 < 3.0.0 #@ filter: pydocstyle < 2.0.0 #@ filter: flake8-docstrings < 1.1.0 #@ filter: flake8-pep3101 < 1.1 -#@ filter: flake8-deprecated < 1.3 diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 11dcfe004..3fb6459a5 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -91,7 +91,7 @@ class Redirect(Exception): self.url = url -class add_handler: # pylint: disable=invalid-name +class add_handler: # noqa: N801,N806 pylint: disable=invalid-name """Decorator to register a qute://* URL handler. diff --git a/qutebrowser/browser/webkit/network/networkmanager.py b/qutebrowser/browser/webkit/network/networkmanager.py index beaa690ca..a19687eb1 100644 --- a/qutebrowser/browser/webkit/network/networkmanager.py +++ b/qutebrowser/browser/webkit/network/networkmanager.py @@ -206,7 +206,7 @@ class NetworkManager(QNetworkAccessManager): # No @pyqtSlot here, see # https://github.com/qutebrowser/qutebrowser/issues/2213 - def on_ssl_errors(self, reply, errors): # pragma: no mccabe + def on_ssl_errors(self, reply, errors): # noqa: C901 pragma: no mccabe """Decide if SSL errors should be ignored or not. This slot is called on SSL/TLS errors by the self.sslErrors signal. diff --git a/qutebrowser/browser/webkit/network/networkreply.py b/qutebrowser/browser/webkit/network/networkreply.py index a4a4f59ca..22263c96b 100644 --- a/qutebrowser/browser/webkit/network/networkreply.py +++ b/qutebrowser/browser/webkit/network/networkreply.py @@ -34,7 +34,7 @@ class FixedDataNetworkReply(QNetworkReply): """QNetworkReply subclass for fixed data.""" - def __init__(self, request, fileData, mimeType, # flake8: disable=N803 + def __init__(self, request, fileData, mimeType, # noqa: N803 parent=None): """Constructor. diff --git a/qutebrowser/commands/cmdutils.py b/qutebrowser/commands/cmdutils.py index 8111a1dd4..2f7af2f9f 100644 --- a/qutebrowser/commands/cmdutils.py +++ b/qutebrowser/commands/cmdutils.py @@ -61,7 +61,7 @@ def check_exclusive(flags, names): argstr)) -class register: # pylint: disable=invalid-name +class register: # noqa: N801,N806 pylint: disable=invalid-name """Decorator to register a new command handler. @@ -114,7 +114,7 @@ class register: # pylint: disable=invalid-name return func -class argument: # pylint: disable=invalid-name +class argument: # noqa: N801,N806 pylint: disable=invalid-name """Decorator to customize an argument for @cmdutils.register. diff --git a/qutebrowser/commands/command.py b/qutebrowser/commands/command.py index bbc79a0d8..afb6253db 100644 --- a/qutebrowser/commands/command.py +++ b/qutebrowser/commands/command.py @@ -393,7 +393,7 @@ class Command: if isinstance(typ, tuple): raise TypeError("{}: Legacy tuple type annotation!".format( self.name)) - elif type(typ) is type(typing.Union): # flake8: disable=E721 + elif type(typ) is type(typing.Union): # noqa: E721 # this is... slightly evil, I know # We also can't use isinstance here because typing.Union doesn't # support that. diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index ff09e2a5b..058f96087 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -38,7 +38,7 @@ key_instance = None change_filters = [] -class change_filter: # pylint: disable=invalid-name +class change_filter: # noqa: N801,N806 pylint: disable=invalid-name """Decorator to filter calls based on a config section/option matching. diff --git a/qutebrowser/keyinput/modeman.py b/qutebrowser/keyinput/modeman.py index cd35913ad..ad9bd06ee 100644 --- a/qutebrowser/keyinput/modeman.py +++ b/qutebrowser/keyinput/modeman.py @@ -62,7 +62,7 @@ class NotInModeError(Exception): def init(win_id, parent): """Initialize the mode manager and the keyparsers for the given win_id.""" - KM = usertypes.KeyMode # pylint: disable=invalid-name + KM = usertypes.KeyMode # noqa: N801,N806 pylint: disable=invalid-name modeman = ModeManager(win_id, parent) objreg.register('mode-manager', modeman, scope='window', window=win_id) keyparsers = { diff --git a/qutebrowser/mainwindow/statusbar/bar.py b/qutebrowser/mainwindow/statusbar/bar.py index 61bb35ace..ae7a3954d 100644 --- a/qutebrowser/mainwindow/statusbar/bar.py +++ b/qutebrowser/mainwindow/statusbar/bar.py @@ -113,7 +113,7 @@ def _generate_stylesheet(): color: {{ conf.colors.%s }}; background-color: {{ conf.colors.%s }}; } - """ % (flag, flag, flag, # flake8: disable=S001 + """ % (flag, flag, flag, # noqa: S001 option + '.fg', option + '.bg') return stylesheet diff --git a/qutebrowser/misc/crashsignal.py b/qutebrowser/misc/crashsignal.py index 68c17d93f..60ef2fbad 100644 --- a/qutebrowser/misc/crashsignal.py +++ b/qutebrowser/misc/crashsignal.py @@ -22,7 +22,7 @@ import os import sys import bdb -import pdb # flake8: disable=T002 +import pdb # noqa: T002 import signal import functools import faulthandler diff --git a/qutebrowser/misc/split.py b/qutebrowser/misc/split.py index 045d9fe4b..bdde2b7ee 100644 --- a/qutebrowser/misc/split.py +++ b/qutebrowser/misc/split.py @@ -55,7 +55,7 @@ class ShellLexer: self.token = '' self.state = ' ' - def __iter__(self): # pragma: no mccabe + def __iter__(self): # noqa: C901 pragma: no mccabe """Read a raw token from the input stream.""" self.reset() for nextchar in self.string: diff --git a/qutebrowser/utils/debug.py b/qutebrowser/utils/debug.py index 5da5234a9..2c86e1142 100644 --- a/qutebrowser/utils/debug.py +++ b/qutebrowser/utils/debug.py @@ -232,7 +232,7 @@ def format_call(func, args=None, kwargs=None, full=True): return '{}({})'.format(name, format_args(args, kwargs)) -class log_time: # pylint: disable=invalid-name +class log_time: # noqa: N801,N806 pylint: disable=invalid-name """Log the time an operation takes. diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py index 0d87a5f7c..b1966d2f7 100644 --- a/qutebrowser/utils/log.py +++ b/qutebrowser/utils/log.py @@ -360,7 +360,7 @@ def qt_message_handler(msg_type, context, msg): # PNGs in Qt with broken color profile # https://bugreports.qt.io/browse/QTBUG-39788 'libpng warning: iCCP: Not recognizing known sRGB profile that has ' - 'been edited', # flake8: disable=E131 + 'been edited', # noqa: E131 'libpng warning: iCCP: known incorrect sRGB profile', # Hopefully harmless warning 'OpenType support missing for script ', @@ -419,7 +419,7 @@ def qt_message_handler(msg_type, context, msg): 'libpng warning: iCCP: known incorrect sRGB profile', # https://bugreports.qt.io/browse/QTBUG-47154 'virtual void QSslSocketBackendPrivate::transmit() SSLRead failed ' - 'with: -9805', # flake8: disable=E131 + 'with: -9805', # noqa: E131 ] if any(msg.strip().startswith(pattern) for pattern in suppressed_msgs): diff --git a/qutebrowser/utils/utils.py b/qutebrowser/utils/utils.py index f59c9f553..8debc6688 100644 --- a/qutebrowser/utils/utils.py +++ b/qutebrowser/utils/utils.py @@ -584,7 +584,7 @@ def disabled_excepthook(): sys.excepthook = old_excepthook -class prevent_exceptions: # pylint: disable=invalid-name +class prevent_exceptions: # noqa: N801,N806 pylint: disable=invalid-name """Decorator to ignore and log exceptions. diff --git a/tests/conftest.py b/tests/conftest.py index 9c52f6800..da0bded55 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,7 +35,7 @@ pytest.register_assert_rewrite('helpers') from helpers import logfail from helpers.logfail import fail_on_logging from helpers.messagemock import message_mock -from helpers.fixtures import * +from helpers.fixtures import * # noqa: F403 from qutebrowser.utils import qtutils, standarddir, usertypes, utils from qutebrowser.misc import objects diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 5cbf01aad..427cdcc0d 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -# pylint: disable=invalid-name +# noqa: N801,N806 pylint: disable=invalid-name """pytest fixtures used by the whole testsuite. diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 878c9e166..9fc04aba3 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -# pylint: disable=invalid-name,abstract-method +# noqa: N801,N806 pylint: disable=invalid-name,abstract-method """Fake objects/stubs.""" diff --git a/tests/unit/commands/test_cmdutils.py b/tests/unit/commands/test_cmdutils.py index e123ce2d2..ca751074d 100644 --- a/tests/unit/commands/test_cmdutils.py +++ b/tests/unit/commands/test_cmdutils.py @@ -103,7 +103,7 @@ class TestRegister: def test_lowercasing(self): """Make sure the function name is normalized correctly (uppercase).""" @cmdutils.register() - def Test(): # pylint: disable=invalid-name + def Test(): # noqa: N801,N806 pylint: disable=invalid-name """Blah.""" pass diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index bf1969e8a..32a7a8119 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -591,7 +591,7 @@ class StyleObj(QObject): def __init__(self, stylesheet=None, parent=None): super().__init__(parent) if stylesheet is not None: - self.STYLESHEET = stylesheet # pylint: disable=invalid-name + self.STYLESHEET = stylesheet # noqa: N801,N806 pylint: disable=invalid-name self.rendered_stylesheet = None def setStyleSheet(self, stylesheet): diff --git a/tests/unit/utils/test_debug.py b/tests/unit/utils/test_debug.py index 9b77b9628..e32f564c8 100644 --- a/tests/unit/utils/test_debug.py +++ b/tests/unit/utils/test_debug.py @@ -252,8 +252,8 @@ class TestGetAllObjects: root = QObject() o1 = self.Object('Object 1', root) - o2 = self.Object('Object 2', o1) # flake8: disable=F841 - o3 = self.Object('Object 3', root) # flake8: disable=F841 + o2 = self.Object('Object 2', o1) # noqa: F841 + o3 = self.Object('Object 3', root) # noqa: F841 expected = textwrap.dedent(""" Qt widgets - 2 objects: From 650b0051e6ae6c8c3e4df1be200d78deaceb17bf Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Sun, 26 Nov 2017 01:59:21 -0600 Subject: [PATCH 124/415] remove useless ignores --- tests/helpers/fixtures.py | 2 +- tests/helpers/stubs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 427cdcc0d..9f4eaa260 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -# noqa: N801,N806 pylint: disable=invalid-name +#pylint: disable=invalid-name """pytest fixtures used by the whole testsuite. diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 9fc04aba3..878c9e166 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -# noqa: N801,N806 pylint: disable=invalid-name,abstract-method +# pylint: disable=invalid-name,abstract-method """Fake objects/stubs.""" From e05dabefdf3e156bae0953686d92bb27ad814ba3 Mon Sep 17 00:00:00 2001 From: Panagiotis Ktistakis Date: Sun, 26 Nov 2017 13:34:18 +0200 Subject: [PATCH 125/415] Show default keybinding in :bind completion --- qutebrowser/completion/models/configmodel.py | 17 +++++++++++++---- qutebrowser/config/config.py | 7 +++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/qutebrowser/completion/models/configmodel.py b/qutebrowser/completion/models/configmodel.py index f1d706cd5..53a2adc19 100644 --- a/qutebrowser/completion/models/configmodel.py +++ b/qutebrowser/completion/models/configmodel.py @@ -77,17 +77,26 @@ def bind(key, *, info): key: the key being bound. """ model = completionmodel.CompletionModel(column_widths=(20, 60, 20)) - cmd_text = info.keyconf.get_command(key, 'normal') + data = [] + cmd_text = info.keyconf.get_command(key, 'normal') if cmd_text: parser = runners.CommandParser() try: cmd = parser.parse(cmd_text).cmd except cmdexc.NoSuchCommandError: - data = [(cmd_text, 'Invalid command!', key)] + data.append((cmd_text, '(Current) Invalid command!', key)) else: - data = [(cmd_text, cmd.desc, key)] - model.add_category(listcategory.ListCategory("Current", data)) + data.append((cmd_text, '(Current) {}'.format(cmd.desc), key)) + + cmd_text = info.keyconf.get_command(key, 'normal', default=True) + if cmd_text: + parser = runners.CommandParser() + cmd = parser.parse(cmd_text).cmd + data.append((cmd_text, '(Default) {}'.format(cmd.desc), key)) + + if data: + model.add_category(listcategory.ListCategory("Current/Default", data)) cmdlist = util.get_cmd_completions(info, include_hidden=True, include_aliases=True) diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py index ff09e2a5b..770546df6 100644 --- a/qutebrowser/config/config.py +++ b/qutebrowser/config/config.py @@ -162,10 +162,13 @@ class KeyConfig: cmd_to_keys[cmd].insert(0, key) return cmd_to_keys - def get_command(self, key, mode): + def get_command(self, key, mode, default=False): """Get the command for a given key (or None).""" key = self._prepare(key, mode) - bindings = self.get_bindings_for(mode) + if default: + bindings = dict(val.bindings.default[mode]) + else: + bindings = self.get_bindings_for(mode) return bindings.get(key, None) def bind(self, key, command, *, mode, save_yaml=False): From 2bb8d404d28cbc628b9f7828a57f6dd648ea38f9 Mon Sep 17 00:00:00 2001 From: Panagiotis Ktistakis Date: Sun, 26 Nov 2017 14:07:41 +0200 Subject: [PATCH 126/415] Adjust :bind completion tests --- tests/unit/completion/test_models.py | 62 +++++++++++++++++++++------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 5276ffd2a..e07c1949c 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -73,6 +73,9 @@ def cmdutils_stub(monkeypatch, stubs): name='scroll', desc='Scroll the current tab in the given direction.', modes=()), + 'tab-close': stubs.FakeCommand( + name='tab-close', + desc='Close the current tab.'), }) @@ -101,9 +104,10 @@ def configdata_stub(monkeypatch, configdata_init): ), ), default={ - 'normal': { - '': 'quit' - } + 'normal': collections.OrderedDict([ + ('', 'quit'), + ('d', 'tab-close'), + ]) }, backends=[], raw_backends=None)), @@ -122,6 +126,7 @@ def configdata_stub(monkeypatch, configdata_init): ('', 'quit'), ('ZQ', 'quit'), ('I', 'invalid'), + ('d', 'scroll down'), ]) }, backends=[], @@ -209,6 +214,7 @@ def test_command_completion(qtmodeltester, cmdutils_stub, configdata_stub, ('open', 'open a url', ''), ('q', "Alias for 'quit'", ''), ('quit', 'quit qutebrowser', 'ZQ, '), + ('tab-close', 'Close the current tab.', ''), ] }) @@ -233,7 +239,8 @@ def test_help_completion(qtmodeltester, cmdutils_stub, key_config_stub, "Commands": [ (':open', 'open a url', ''), (':quit', 'quit qutebrowser', 'ZQ, '), - (':scroll', 'Scroll the current tab in the given direction.', '') + (':scroll', 'Scroll the current tab in the given direction.', ''), + (':tab-close', 'Close the current tab.', ''), ], "Settings": [ ('aliases', 'Aliases for commands.', None), @@ -566,9 +573,9 @@ def test_setting_option_completion(qtmodeltester, config_stub, ('aliases', 'Aliases for commands.', '{"q": "quit"}'), ('bindings.commands', 'Default keybindings', '{"normal": {"": "quit", "ZQ": "quit", ' - '"I": "invalid"}}'), + '"I": "invalid", "d": "scroll down"}}'), ('bindings.default', 'Default keybindings', - '{"normal": {"": "quit"}}'), + '{"normal": {"": "quit", "d": "tab-close"}}'), ] }) @@ -589,14 +596,15 @@ def test_bind_completion(qtmodeltester, cmdutils_stub, config_stub, qtmodeltester.check(model) _check_completions(model, { - "Current": [ - ('quit', 'quit qutebrowser', 'ZQ'), + "Current/Default": [ + ('quit', '(Current) quit qutebrowser', 'ZQ'), ], "Commands": [ ('open', 'open a url', ''), ('q', "Alias for 'quit'", ''), ('quit', 'quit qutebrowser', 'ZQ, '), - ('scroll', 'Scroll the current tab in the given direction.', '') + ('scroll', 'Scroll the current tab in the given direction.', ''), + ('tab-close', 'Close the current tab.', ''), ], }) @@ -608,21 +616,22 @@ def test_bind_completion_invalid(cmdutils_stub, config_stub, key_config_stub, model.set_pattern('') _check_completions(model, { - "Current": [ - ('invalid', 'Invalid command!', 'I'), + "Current/Default": [ + ('invalid', '(Current) Invalid command!', 'I'), ], "Commands": [ ('open', 'open a url', ''), ('q', "Alias for 'quit'", ''), ('quit', 'quit qutebrowser', 'ZQ, '), - ('scroll', 'Scroll the current tab in the given direction.', '') + ('scroll', 'Scroll the current tab in the given direction.', ''), + ('tab-close', 'Close the current tab.', ''), ], }) -def test_bind_completion_no_current(qtmodeltester, cmdutils_stub, config_stub, +def test_bind_completion_no_binding(qtmodeltester, cmdutils_stub, config_stub, key_config_stub, configdata_stub, info): - """Test keybinding completion with no current binding.""" + """Test keybinding completion with no current or default binding.""" model = configmodel.bind('x', info=info) model.set_pattern('') qtmodeltester.data_display_may_return_none = True @@ -633,7 +642,30 @@ def test_bind_completion_no_current(qtmodeltester, cmdutils_stub, config_stub, ('open', 'open a url', ''), ('q', "Alias for 'quit'", ''), ('quit', 'quit qutebrowser', 'ZQ, '), - ('scroll', 'Scroll the current tab in the given direction.', '') + ('scroll', 'Scroll the current tab in the given direction.', ''), + ('tab-close', 'Close the current tab.', ''), + ], + }) + + +def test_bind_completion_changed(cmdutils_stub, config_stub, key_config_stub, + configdata_stub, info): + """Test command completion with a non-default command bound.""" + model = configmodel.bind('d', info=info) + model.set_pattern('') + + _check_completions(model, { + "Current/Default": [ + ('scroll down', + '(Current) Scroll the current tab in the given direction.', 'd'), + ('tab-close', '(Default) Close the current tab.', 'd'), + ], + "Commands": [ + ('open', 'open a url', ''), + ('q', "Alias for 'quit'", ''), + ('quit', 'quit qutebrowser', 'ZQ, '), + ('scroll', 'Scroll the current tab in the given direction.', ''), + ('tab-close', 'Close the current tab.', ''), ], }) From 54fffc826450a8c70e7dacb068590e45ef3358ce Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 24 Nov 2017 15:15:03 -0500 Subject: [PATCH 127/415] Resolve crash when editor tab is closed. If an editor is open on a form in a tab and that tab is closed, rewire the callback to print a warning. Previously, the callback would access a deleted C++ object and cause a crash. Resolves #2758. --- qutebrowser/browser/commands.py | 8 ++++++++ tests/end2end/features/editor.feature | 10 ++++++++++ tests/end2end/features/test_editor_bdd.py | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index dc91a321e..3dcece045 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1621,6 +1621,9 @@ class CommandDispatcher: ed = editor.ExternalEditor(self._tabbed_browser) ed.editing_finished.connect(functools.partial( self.on_editing_finished, elem)) + tab = self._current_widget() + tab.shutting_down.connect(functools.partial( + self.on_editor_orphaned, ed)) ed.edit(text, caret_position) @cmdutils.register(instance='command-dispatcher', scope='window') @@ -1647,6 +1650,11 @@ class CommandDispatcher: except webelem.Error as e: raise cmdexc.CommandError(str(e)) + def on_editor_orphaned(self, ed): + ed.editing_finished.disconnect() + ed.editing_finished.connect( + lambda: message.warning('Edited element was closed')) + @cmdutils.register(instance='command-dispatcher', maxsplit=0, scope='window') def insert_text(self, text): diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 93b61df30..92de6ccdd 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -115,6 +115,16 @@ Feature: Opening external editors And I run :click-element id qute-button Then the javascript message "text: foobar" should be logged + Scenario: Spawning an editor and closing the tab + When I set up a fake editor returning "foo" after 1s + And I open data/editor.html + And I run :click-element id qute-textarea + And I wait for "Entering mode KeyMode.insert (reason: clicking input)" in the log + And I run :open-editor + And I set tabs.last_close to blank + And I run :tab-close + Then the warning "Edited element was closed" should be shown + @qtwebengine_todo: Caret mode is not implemented yet Scenario: Spawning an editor in caret mode When I set up a fake editor returning "foobar" diff --git a/tests/end2end/features/test_editor_bdd.py b/tests/end2end/features/test_editor_bdd.py index f66b94f77..0cd85a351 100644 --- a/tests/end2end/features/test_editor_bdd.py +++ b/tests/end2end/features/test_editor_bdd.py @@ -64,3 +64,21 @@ def set_up_editor(quteproc, server, tmpdir, text): def set_up_editor_empty(quteproc, server, tmpdir): """Set up editor.command to a small python script inserting empty text.""" set_up_editor(quteproc, server, tmpdir, "") + + +@bdd.when(bdd.parsers.parse( + 'I set up a fake editor returning "{text}" after {t}s')) +def set_up_editor_delay(quteproc, server, tmpdir, text, t): + """Set up editor.command to a small python script inserting a text.""" + script = tmpdir / 'script.py' + script.write(textwrap.dedent(""" + import sys + import time + + time.sleep({t}) + + with open(sys.argv[1], 'w', encoding='utf-8') as f: + f.write({text!r}) + """.format(text=text, t=t))) + editor = json.dumps([sys.executable, str(script), '{}']) + quteproc.set_setting('editor.command', editor) From 67253726fa82471abb0ac4e2e5ae5168b2f6575c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 26 Nov 2017 17:31:07 +0100 Subject: [PATCH 128/415] Handle empty messages in qt_message_handler I can't reproduce this, but someone on KDE reported always getting a crash (as msg.splitlines()[0] gives an IndexError) when trying to select a file with Qt 5.9.3. --- qutebrowser/utils/log.py | 3 +++ tests/unit/utils/test_log.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py index 0d87a5f7c..97013a82b 100644 --- a/qutebrowser/utils/log.py +++ b/qutebrowser/utils/log.py @@ -422,6 +422,9 @@ def qt_message_handler(msg_type, context, msg): 'with: -9805', # flake8: disable=E131 ] + if not msg: + msg = "Logged empty message!" + if any(msg.strip().startswith(pattern) for pattern in suppressed_msgs): level = logging.DEBUG else: diff --git a/tests/unit/utils/test_log.py b/tests/unit/utils/test_log.py index 147e760bb..30dd5d634 100644 --- a/tests/unit/utils/test_log.py +++ b/tests/unit/utils/test_log.py @@ -25,12 +25,15 @@ import itertools import sys import warnings +import attr import pytest import pytest_catchlog from qutebrowser.utils import log from qutebrowser.misc import utilcmds +from PyQt5 import QtCore + @pytest.fixture(autouse=True) def restore_loggers(): @@ -252,3 +255,22 @@ def test_ignore_py_warnings(caplog): assert len(caplog.records) == 1 msg = caplog.records[0].message.splitlines()[0] assert msg.endswith("UserWarning: not hidden") + + +class TestQtMessageHandler: + + @attr.s + class Context: + + """Fake QMessageLogContext.""" + + function = attr.ib(default=None) + category = attr.ib(default=None) + file = attr.ib(default=None) + line = attr.ib(default=None) + + def test_empty_message(self, caplog): + """Make sure there's no crash with an empty message.""" + log.qt_message_handler(QtCore.QtDebugMsg, self.Context(), "") + assert len(caplog.records) == 1 + assert caplog.records[0].msg == "Logged empty message!" From 4c1f6158bdf960357947f6de67efc8d059e51df4 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 26 Nov 2017 17:32:29 +0100 Subject: [PATCH 129/415] Update changelog [ci skip] --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index cb7a7ba37..f507ea803 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -128,6 +128,7 @@ Fixed - The `qute://gpl` page now works correctly again. - Trying to bind an empty command now doesn't crash anymore. - Fixed crash when `:config-write-py` fails to write to the given path. +- Fixed crash for some users when selecting a file with Qt 5.9.3 v1.0.3 ------ From 95f34d755f8104b5a906efc7480a423c1ab8b82d Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 26 Nov 2017 19:35:58 +0100 Subject: [PATCH 130/415] Update changelog --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index f507ea803..e06ddb6e3 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -87,6 +87,7 @@ Changed a line of code. - `:jseval --file` now searches relative paths in a js/ subdir in qutebrowser's data dir, e.g. `~/.local/share/qutebrowser/js`. +- The current/default bindings are now shown in the :bind completion. Fixed ~~~~~ From 7520a365eb59dcd0c8199722b7639abd05ab17b5 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 26 Nov 2017 19:43:49 +0100 Subject: [PATCH 131/415] Update comments --- misc/requirements/requirements-flake8.txt-raw | 2 +- tests/helpers/fixtures.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/requirements/requirements-flake8.txt-raw b/misc/requirements/requirements-flake8.txt-raw index 2baced0d0..2878edfba 100644 --- a/misc/requirements/requirements-flake8.txt-raw +++ b/misc/requirements/requirements-flake8.txt-raw @@ -14,7 +14,7 @@ pep8-naming pydocstyle<2.0.0 pyflakes -# Waiting until flake8-putty updated +# Not upgraded yet because they introduce new errors #@ filter: pydocstyle < 2.0.0 #@ filter: flake8-docstrings < 1.1.0 #@ filter: flake8-pep3101 < 1.1 diff --git a/tests/helpers/fixtures.py b/tests/helpers/fixtures.py index 9f4eaa260..5cbf01aad 100644 --- a/tests/helpers/fixtures.py +++ b/tests/helpers/fixtures.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -#pylint: disable=invalid-name +# pylint: disable=invalid-name """pytest fixtures used by the whole testsuite. From b5dd64767846d67161c3c15125c57b2d9951352a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 26 Nov 2017 19:50:13 +0100 Subject: [PATCH 132/415] Upgrade pydocstyle/flake8-docstrings See #3320 --- .flake8 | 6 +++++- misc/requirements/requirements-flake8.txt | 5 +++-- misc/requirements/requirements-flake8.txt-raw | 6 ++---- qutebrowser/browser/qtnetworkdownloads.py | 2 +- qutebrowser/commands/runners.py | 1 - scripts/dev/src2asciidoc.py | 2 +- tests/unit/browser/webkit/http/test_content_disposition.py | 2 +- tests/unit/misc/test_cmdhistory.py | 4 ++-- 8 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.flake8 b/.flake8 index e29e7037c..61fdc0326 100644 --- a/.flake8 +++ b/.flake8 @@ -19,18 +19,22 @@ exclude = .*,__pycache__,resources.py # D103: Missing docstring in public function (will be handled by others) # D104: Missing docstring in public package (will be handled by others) # D105: Missing docstring in magic method (will be handled by others) +# D106: Missing docstring in public nested class (will be handled by others) +# D107: Missing docstring in __init__ (will be handled by others) # D209: Blank line before closing """ (removed from PEP257) # D211: No blank lines allowed before class docstring # (PEP257 got changed, but let's stick to the old standard) +# D401: First line should be in imperative mood (okay sometimes) # D402: First line should not be function's signature (false-positives) # D403: First word of the first line should be properly capitalized # (false-positives) +# D413: Missing blank line after last section (not in pep257?) ignore = E128,E226,E265,E501,E402,E266,E722,E731, F401, N802, P101,P102,P103, - D102,D103,D104,D105,D209,D211,D402,D403 + D102,D103,D106,D107,D104,D105,D209,D211,D401,D402,D403,D413 min-version = 3.4.0 max-complexity = 12 per-file-ignores = diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index ff33c19e1..b34ce5b67 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -4,7 +4,7 @@ flake8==3.5.0 flake8-copyright==0.2.0 flake8-debugger==3.0.0 flake8-deprecated==1.3 -flake8-docstrings==1.0.3 # rq.filter: < 1.1.0 +flake8-docstrings==1.1.0 flake8-future-import==0.4.3 flake8-mock==0.3 flake8-pep3101==1.0 # rq.filter: < 1.1 @@ -16,6 +16,7 @@ flake8-tuple==0.2.13 mccabe==0.6.1 pep8-naming==0.4.1 pycodestyle==2.3.1 -pydocstyle==1.1.1 # rq.filter: < 2.0.0 +pydocstyle==2.1.1 pyflakes==1.6.0 six==1.11.0 +snowballstemmer==1.2.1 diff --git a/misc/requirements/requirements-flake8.txt-raw b/misc/requirements/requirements-flake8.txt-raw index 2878edfba..8abcc51c7 100644 --- a/misc/requirements/requirements-flake8.txt-raw +++ b/misc/requirements/requirements-flake8.txt-raw @@ -2,7 +2,7 @@ flake8 flake8-copyright flake8-debugger flake8-deprecated -flake8-docstrings<1.1.0 +flake8-docstrings flake8-future-import flake8-mock flake8-pep3101<1.1 @@ -11,10 +11,8 @@ flake8-string-format flake8-tidy-imports flake8-tuple pep8-naming -pydocstyle<2.0.0 +pydocstyle pyflakes # Not upgraded yet because they introduce new errors -#@ filter: pydocstyle < 2.0.0 -#@ filter: flake8-docstrings < 1.1.0 #@ filter: flake8-pep3101 < 1.1 diff --git a/qutebrowser/browser/qtnetworkdownloads.py b/qutebrowser/browser/qtnetworkdownloads.py index 709c8207b..0e079b633 100644 --- a/qutebrowser/browser/qtnetworkdownloads.py +++ b/qutebrowser/browser/qtnetworkdownloads.py @@ -215,7 +215,7 @@ class DownloadItem(downloads.AbstractDownloadItem): abort_on=[self.cancelled, self.error]) def _set_fileobj(self, fileobj, *, autoclose=True): - """"Set the file object to write the download to. + """Set the file object to write the download to. Args: fileobj: A file-like object. diff --git a/qutebrowser/commands/runners.py b/qutebrowser/commands/runners.py index fadb6c063..095b49108 100644 --- a/qutebrowser/commands/runners.py +++ b/qutebrowser/commands/runners.py @@ -95,7 +95,6 @@ class CommandParser: """Parse qutebrowser commandline commands. Attributes: - _partial_match: Whether to allow partial command matches. """ diff --git a/scripts/dev/src2asciidoc.py b/scripts/dev/src2asciidoc.py index a0ddfb795..1470103ab 100755 --- a/scripts/dev/src2asciidoc.py +++ b/scripts/dev/src2asciidoc.py @@ -386,7 +386,7 @@ def generate_commands(filename): def _generate_setting_backend_info(f, opt): - """"Generate backend information for the given option.""" + """Generate backend information for the given option.""" all_backends = [usertypes.Backend.QtWebKit, usertypes.Backend.QtWebEngine] if opt.raw_backends is not None: for name, conditional in sorted(opt.raw_backends.items()): diff --git a/tests/unit/browser/webkit/http/test_content_disposition.py b/tests/unit/browser/webkit/http/test_content_disposition.py index e1f78eb74..5aa25166e 100644 --- a/tests/unit/browser/webkit/http/test_content_disposition.py +++ b/tests/unit/browser/webkit/http/test_content_disposition.py @@ -581,7 +581,7 @@ class TestAttachment: header_checker.check_ignored('attachment; filename=bar foo=foo') def test_attmissingdelim3(self, header_checker): - """";" missing between disposition type and filename parameter. + """';' missing between disposition type and filename parameter. This is invalid, so UAs should ignore it. """ diff --git a/tests/unit/misc/test_cmdhistory.py b/tests/unit/misc/test_cmdhistory.py index 7858831b6..44c6c2d82 100644 --- a/tests/unit/misc/test_cmdhistory.py +++ b/tests/unit/misc/test_cmdhistory.py @@ -129,14 +129,14 @@ def test_nextitem_previtem_chain(hist): def test_nextitem_index_error(hist): - """"Test nextitem() when _tmphist raises an IndexError.""" + """Test nextitem() when _tmphist raises an IndexError.""" hist.start('f') with pytest.raises(cmdhistory.HistoryEndReachedError): hist.nextitem() def test_previtem_index_error(hist): - """"Test previtem() when _tmphist raises an IndexError.""" + """Test previtem() when _tmphist raises an IndexError.""" hist.start('f') with pytest.raises(cmdhistory.HistoryEndReachedError): for _ in range(10): From 5490f70b25341978b5ffff0c82b397abfb01ec40 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 26 Nov 2017 20:30:10 +0100 Subject: [PATCH 133/415] Remove flake8-pep3101 It seems to trigger when using modulo on ints as well, which is kind of annoying. See #3320 --- misc/requirements/requirements-flake8.txt | 1 - misc/requirements/requirements-flake8.txt-raw | 4 ---- 2 files changed, 5 deletions(-) diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index b34ce5b67..5c4e97e4e 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -7,7 +7,6 @@ flake8-deprecated==1.3 flake8-docstrings==1.1.0 flake8-future-import==0.4.3 flake8-mock==0.3 -flake8-pep3101==1.0 # rq.filter: < 1.1 flake8-per-file-ignores==0.4 flake8-polyfill==1.0.1 flake8-string-format==0.2.3 diff --git a/misc/requirements/requirements-flake8.txt-raw b/misc/requirements/requirements-flake8.txt-raw index 8abcc51c7..6836bb979 100644 --- a/misc/requirements/requirements-flake8.txt-raw +++ b/misc/requirements/requirements-flake8.txt-raw @@ -5,7 +5,6 @@ flake8-deprecated flake8-docstrings flake8-future-import flake8-mock -flake8-pep3101<1.1 flake8-per-file-ignores flake8-string-format flake8-tidy-imports @@ -13,6 +12,3 @@ flake8-tuple pep8-naming pydocstyle pyflakes - -# Not upgraded yet because they introduce new errors -#@ filter: flake8-pep3101 < 1.1 From 1981239478f21faa962fc49b99eb4076b6ee1190 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 26 Nov 2017 21:22:33 +0100 Subject: [PATCH 134/415] Add flake8-builtins --- .flake8 | 4 +++- misc/requirements/requirements-flake8.txt | 1 + misc/requirements/requirements-flake8.txt-raw | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 61fdc0326..74f68c27a 100644 --- a/.flake8 +++ b/.flake8 @@ -29,12 +29,14 @@ exclude = .*,__pycache__,resources.py # D403: First word of the first line should be properly capitalized # (false-positives) # D413: Missing blank line after last section (not in pep257?) +# A003: Builtin name for class attribute (needed for attrs) ignore = E128,E226,E265,E501,E402,E266,E722,E731, F401, N802, P101,P102,P103, - D102,D103,D106,D107,D104,D105,D209,D211,D401,D402,D403,D413 + D102,D103,D106,D107,D104,D105,D209,D211,D401,D402,D403,D413, + A003 min-version = 3.4.0 max-complexity = 12 per-file-ignores = diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index 5c4e97e4e..661f9601e 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -1,6 +1,7 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py flake8==3.5.0 +flake8-builtins==1.0 flake8-copyright==0.2.0 flake8-debugger==3.0.0 flake8-deprecated==1.3 diff --git a/misc/requirements/requirements-flake8.txt-raw b/misc/requirements/requirements-flake8.txt-raw index 6836bb979..0bfdab644 100644 --- a/misc/requirements/requirements-flake8.txt-raw +++ b/misc/requirements/requirements-flake8.txt-raw @@ -1,4 +1,5 @@ flake8 +flake8-builtins flake8-copyright flake8-debugger flake8-deprecated From 248a12a8b91e15ba0a997daad23f0fe751f3a9d6 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 26 Nov 2017 21:42:50 +0100 Subject: [PATCH 135/415] Add flake8-comprehensions --- misc/requirements/requirements-flake8.txt | 1 + misc/requirements/requirements-flake8.txt-raw | 1 + qutebrowser/browser/webelem.py | 2 +- qutebrowser/browser/webkit/mhtml.py | 4 ++-- qutebrowser/utils/usertypes.py | 4 ++-- scripts/dev/cleanup.py | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index 661f9601e..26995cb59 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -2,6 +2,7 @@ flake8==3.5.0 flake8-builtins==1.0 +flake8-comprehensions==1.4.1 flake8-copyright==0.2.0 flake8-debugger==3.0.0 flake8-deprecated==1.3 diff --git a/misc/requirements/requirements-flake8.txt-raw b/misc/requirements/requirements-flake8.txt-raw index 0bfdab644..30f6c28d8 100644 --- a/misc/requirements/requirements-flake8.txt-raw +++ b/misc/requirements/requirements-flake8.txt-raw @@ -1,5 +1,6 @@ flake8 flake8-builtins +flake8-comprehensions flake8-copyright flake8-debugger flake8-deprecated diff --git a/qutebrowser/browser/webelem.py b/qutebrowser/browser/webelem.py index 067f33cff..36479174f 100644 --- a/qutebrowser/browser/webelem.py +++ b/qutebrowser/browser/webelem.py @@ -221,7 +221,7 @@ class AbstractWebElement(collections.abc.MutableMapping): } relevant_classes = classes[self.tag_name()] for klass in self.classes(): - if any([klass.strip().startswith(e) for e in relevant_classes]): + if any(klass.strip().startswith(e) for e in relevant_classes): return True return False diff --git a/qutebrowser/browser/webkit/mhtml.py b/qutebrowser/browser/webkit/mhtml.py index ccdd03dad..67c8a5b7a 100644 --- a/qutebrowser/browser/webkit/mhtml.py +++ b/qutebrowser/browser/webkit/mhtml.py @@ -502,8 +502,8 @@ class _Downloader: This is needed if a download finishes before attaching its finished signal. """ - items = set((url, item) for url, item in self.pending_downloads - if item.done) + items = {(url, item) for url, item in self.pending_downloads + if item.done} log.downloads.debug("Zombie downloads: {}".format(items)) for url, item in items: self._finished(url, item) diff --git a/qutebrowser/utils/usertypes.py b/qutebrowser/utils/usertypes.py index 99716c062..aad685d07 100644 --- a/qutebrowser/utils/usertypes.py +++ b/qutebrowser/utils/usertypes.py @@ -102,8 +102,8 @@ class NeighborList(collections.abc.Sequence): if items: item = min(items, key=lambda tpl: abs(self.fuzzyval - tpl[1])) else: - sorted_items = sorted([(idx, e) for (idx, e) in - enumerate(self.items)], key=lambda e: e[1]) + sorted_items = sorted(((idx, e) for (idx, e) in + enumerate(self.items)), key=lambda e: e[1]) idx = 0 if offset < 0 else -1 item = sorted_items[idx] self._idx = item[0] diff --git a/scripts/dev/cleanup.py b/scripts/dev/cleanup.py index 297b876c0..086b210ab 100755 --- a/scripts/dev/cleanup.py +++ b/scripts/dev/cleanup.py @@ -61,7 +61,7 @@ def main(): for root, _dirs, _files in os.walk(os.getcwd()): path = os.path.basename(root) - if any([fnmatch.fnmatch(path, e) for e in recursive_lint]): + if any(fnmatch.fnmatch(path, e) for e in recursive_lint): remove(root) From 75a8938e83709073b182fb2e0d197e758c7c0def Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 26 Nov 2017 21:49:37 +0100 Subject: [PATCH 136/415] Add flake8-bugbear --- .flake8 | 3 +++ misc/requirements/requirements-flake8.txt | 2 ++ misc/requirements/requirements-flake8.txt-raw | 1 + scripts/dev/run_vulture.py | 2 +- tests/unit/completion/test_completionwidget.py | 2 +- 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index 74f68c27a..06883f526 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,7 @@ [flake8] exclude = .*,__pycache__,resources.py +# B001: bare except +# B305: .next() (false-positives) # E128: continuation line under-indented for visual indent # E226: missing whitespace around arithmetic operator # E265: Block comment should start with '#' @@ -31,6 +33,7 @@ exclude = .*,__pycache__,resources.py # D413: Missing blank line after last section (not in pep257?) # A003: Builtin name for class attribute (needed for attrs) ignore = + B001,B305, E128,E226,E265,E501,E402,E266,E722,E731, F401, N802, diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index 26995cb59..a031778ba 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -1,6 +1,8 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py +attrs==17.3.0 flake8==3.5.0 +flake8-bugbear==17.4.0 flake8-builtins==1.0 flake8-comprehensions==1.4.1 flake8-copyright==0.2.0 diff --git a/misc/requirements/requirements-flake8.txt-raw b/misc/requirements/requirements-flake8.txt-raw index 30f6c28d8..1f30b83ae 100644 --- a/misc/requirements/requirements-flake8.txt-raw +++ b/misc/requirements/requirements-flake8.txt-raw @@ -1,4 +1,5 @@ flake8 +flake8-bugbear flake8-builtins flake8-comprehensions flake8-copyright diff --git a/scripts/dev/run_vulture.py b/scripts/dev/run_vulture.py index b4d064c0d..9d21ad428 100755 --- a/scripts/dev/run_vulture.py +++ b/scripts/dev/run_vulture.py @@ -103,7 +103,7 @@ def whitelist_generator(): # noqa for attr in ['visit_call', 'process_module']: yield 'scripts.dev.pylint_checkers.modeline.' + attr - for name, member in inspect.getmembers(configtypes, inspect.isclass): + for name, _member in inspect.getmembers(configtypes, inspect.isclass): yield 'qutebrowser.config.configtypes.' + name yield 'qutebrowser.config.configexc.ConfigErrorDesc.traceback' yield 'qutebrowser.config.configfiles.ConfigAPI.load_autoconfig' diff --git a/tests/unit/completion/test_completionwidget.py b/tests/unit/completion/test_completionwidget.py index b03205da7..a639daf95 100644 --- a/tests/unit/completion/test_completionwidget.py +++ b/tests/unit/completion/test_completionwidget.py @@ -45,7 +45,7 @@ def completionview(qtbot, status_command_stub, config_stub, win_registry, def test_set_model(completionview): """Ensure set_model actually sets the model and expands all categories.""" model = completionmodel.CompletionModel() - for i in range(3): + for _i in range(3): model.add_category(listcategory.ListCategory('', [('foo',)])) completionview.set_model(model) assert completionview.model() is model From 568d60753e4856e9f84921e1084ea23d4e9e5956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andor=20Uhl=C3=A1r?= Date: Sun, 10 May 2015 18:33:36 +0200 Subject: [PATCH 137/415] Add greasemonkey compatible userscript module. WebKit backend only for now. Loads all .js files from a directory, specified in the greasemonkey-directory key in the storage section, defaulting to data/greasemonkey, and wraps them in a minimal environment providing some GM_* functions. Makes those scripts available via the "greasemonkey" registered object in objreg and injects scripts at appropriate times in a page load base on @run-at directives. --- qutebrowser/browser/browsertab.py | 6 +- qutebrowser/browser/greasemonkey.py | 262 ++++++++++++++++++++++++++ qutebrowser/browser/webkit/webpage.py | 28 +++ qutebrowser/utils/log.py | 4 +- 4 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 qutebrowser/browser/greasemonkey.py diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index 547e276db..866943f87 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -31,7 +31,7 @@ from qutebrowser.keyinput import modeman from qutebrowser.config import config from qutebrowser.utils import utils, objreg, usertypes, log, qtutils from qutebrowser.misc import miscwidgets, objects -from qutebrowser.browser import mouse, hints +from qutebrowser.browser import mouse, hints, greasemonkey tab_id_gen = itertools.count(0) @@ -64,6 +64,10 @@ def init(): from qutebrowser.browser.webengine import webenginetab webenginetab.init() + log.init.debug("Initializing Greasemonkey...") + gm_manager = greasemonkey.GreasemonkeyManager() + objreg.register('greasemonkey', gm_manager) + class WebTabError(Exception): diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py new file mode 100644 index 000000000..d1f63a4a4 --- /dev/null +++ b/qutebrowser/browser/greasemonkey.py @@ -0,0 +1,262 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Load, parse and make avalaible greasemonkey scripts.""" + +import re +import os +import json +import fnmatch +import functools +import glob + +from PyQt5.QtCore import pyqtSignal, QObject + +from qutebrowser.utils import log, standarddir + +# TODO: GM_ bootstrap + + +def _scripts_dir(): + """Get the directory of the scripts.""" + return os.path.join(standarddir.data(), 'greasemonkey') + + +class GreasemonkeyScript: + """Container class for userscripts, parses metadata blocks.""" + + GM_BOOTSTRAP_TEMPLATE = r"""var _qute_script_id = "__gm_{scriptName}"; + +function GM_log(text) {{ + console.log(text); +}} + +GM_info = (function() {{ + return {{ + 'script': {scriptInfo}, + 'scriptMetaStr': {scriptMeta}, + 'scriptWillUpdate': false, + 'version': '0.0.1', + 'scriptHandler': 'Tampermonkey' //so scripts don't expect exportFunction + }}; +}}()); + +function GM_setValue(key, value) {{ + if (localStorage !== null && + typeof key === "string" && + (typeof value === "string" || + typeof value === "number" || + typeof value == "boolean")) {{ + localStorage.setItem(_qute_script_id + key, value); + }} +}} + +function GM_getValue(key, default_) {{ + if (localStorage !== null && typeof key === "string") {{ + return localStorage.getItem(_qute_script_id + key) || default_; + }} +}} + +function GM_deleteValue(key) {{ + if (localStorage !== null && typeof key === "string") {{ + localStorage.removeItem(_qute_script_id + key); + }} +}} + +function GM_listValues() {{ + var i; + var keys = []; + for (i = 0; i < localStorage.length; ++i) {{ + if (localStorage.key(i).startsWith(_qute_script_id)) {{ + keys.push(localStorage.key(i)); + }} + }} + return keys; +}} + +function GM_openInTab(url) {{ + window.open(url); +}} + + +// Almost verbatim copy from Eric +function GM_xmlhttpRequest(/* object */ details) {{ + details.method = details.method.toUpperCase() || "GET"; + + if(!details.url) {{ + throw("GM_xmlhttpRequest requires an URL."); + }} + + // build XMLHttpRequest object + var oXhr = new XMLHttpRequest; + // run it + if("onreadystatechange" in details) + oXhr.onreadystatechange = function() {{ + details.onreadystatechange(oXhr) + }}; + if("onload" in details) + oXhr.onload = function() {{ details.onload(oXhr) }}; + if("onerror" in details) + oXhr.onerror = function() {{ details.onerror(oXhr) }}; + + oXhr.open(details.method, details.url, true); + + if("headers" in details) + for(var header in details.headers) + oXhr.setRequestHeader(header, details.headers[header]); + + if("data" in details) + oXhr.send(details.data); + else + oXhr.send(); +}} + +function GM_addStyle(/* String */ styles) {{ + var head = document.getElementsByTagName("head")[0]; + if (head === undefined) {{ + document.onreadystatechange = function() {{ + if (document.readyState == "interactive") {{ + var oStyle = document.createElement("style"); + oStyle.setAttribute("type", "text/css"); + oStyle.appendChild(document.createTextNode(styles)); + document.getElementsByTagName("head")[0].appendChild(oStyle); + }} + }} + }} + else {{ + var oStyle = document.createElement("style"); + oStyle.setAttribute("type", "text/css"); + oStyle.appendChild(document.createTextNode(styles)); + head.appendChild(oStyle); + }} +}} + +unsafeWindow = window; +""" + + def __init__(self, properties, code): + self._code = code + self.includes = [] + self.excludes = [] + self.description = None + self.name = None + self.run_at = None + for name, value in properties: + if name == 'name': + self.name = value + elif name == 'description': + self.description = value + elif name in ['include', 'match']: + self.includes.append(value) + elif name in ['exclude', 'exclude_match']: + self.excludes.append(value) + elif name == 'run-at': + self.run_at = value + + HEADER_REGEX = r'// ==UserScript==.|\n+// ==/UserScript==\n' + PROPS_REGEX = r'// @(?P[^\s]+)\s+(?P.+)' + + @classmethod + def parse(cls, source): + """GreaseMonkeyScript factory. + + Takes a userscript source and returns a GreaseMonkeyScript. + Parses the greasemonkey metadata block, if present, to fill out + attributes. + """ + matches = re.split(cls.HEADER_REGEX, source, maxsplit=1) + try: + props, _code = matches + except ValueError: + props = "" + script = cls(re.findall(cls.PROPS_REGEX, props), code) + script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:])) + return script + + def code(self): + """Return the processed javascript code of this script. + + Adorns the source code with GM_* methods for greasemonkey + compatibility and wraps it in an IFFE to hide it within a + lexical scope. Note that this means line numbers in your + browser's debugger/inspector will not match up to the line + numbers in the source script directly. + """ + gm_bootstrap = self.GM_BOOTSTRAP_TEMPLATE.format( + scriptName=self.name, + scriptInfo=self._meta_json(), + scriptMeta=self.script_meta) + return '\n'.join([gm_bootstrap, self._code]) + + def _meta_json(self): + return json.dumps({ + 'name': self.name, + 'description': self.description, + 'matches': self.includes, + 'includes': self.includes, + 'excludes': self.excludes, + 'run-at': self.run_at, + }) + + +class GreasemonkeyManager: + + def __init__(self, parent=None): + super().__init__(parent) + self._run_start = [] + self._run_end = [] + + scripts_dir = os.path.abspath(_scripts_dir()) + log.greasemonkey.debug("Reading scripts from: {}".format(scripts_dir)) + for script_filename in glob.glob(os.path.join(scripts_dir, '*.js')): + if not os.path.isfile(script_filename): + continue + script_path = os.path.join(scripts_dir, script_filename) + with open(script_path, encoding='utf-8') as script_file: + script = GreasemonkeyScript.parse(script_file.read()) + if not script.name: + script.name = script_filename + + if script.run_at == 'document-start': + self._run_start.append(script) + elif script.run_at == 'document-end': + self._run_end.append(script) + else: + log.greasemonkey.warning("Script {} has invalid run-at " + "defined, ignoring." + .format(script_path)) + continue + log.greasemonkey.debug("Loaded script: {}".format(script.name)) + + def scripts_for(self, url): + """Fetch scripts that are registered to run for url. + + returns a tuple of lists of scripts meant to run at (document-start, + document-end) + """ + match = functools.partial(fnmatch.fnmatch, url) + tester = (lambda script: + any(map(match, script.includes())) and not + any(map(match, script.excludes()))) + return (list(filter(tester, self._run_start)), + list(filter(tester, self._run_end))) + + def all_scripts(self): + """Return all scripts found in the configured script directory.""" + return self._run_start + self._run_end diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 7e1d991b9..9beba6ddc 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -86,6 +86,10 @@ class BrowserPage(QWebPage): self.on_save_frame_state_requested) self.restoreFrameStateRequested.connect( self.on_restore_frame_state_requested) + self.mainFrame().javaScriptWindowObjectCleared.connect( + functools.partial(self.inject_userjs, load='start')) + self.mainFrame().initialLayoutCompleted.connect( + functools.partial(self.inject_userjs, load='end')) def javaScriptPrompt(self, frame, js_msg, default): """Override javaScriptPrompt to use qutebrowser prompts.""" @@ -283,6 +287,30 @@ class BrowserPage(QWebPage): else: self.error_occurred = False + @pyqtSlot() + def inject_userjs(self, load): + """Inject user javascripts into the page. + + param: The page load stage to inject the corresponding scripts + for. Support values are "start" and "end", + corresponding to the allowed values of the `@run-at` + directive in the greasemonkey metadata spec. + """ + greasemonkey = objreg.get('greasemonkey') + url = self.currentFrame().url() + start_scripts, end_scripts = greasemonkey.scripts_for(url.toDisplayString()) + log.greasemonkey.debug('scripts: {}'.format(start_scripts if start else end_scripts)) + + toload = [] + if load == "start": + toload = start_scripts + elif load == "end": + toload = end_scripts + + for script in toload: + log.webview.debug('Running GM script: {}'.format(script.name)) + self.currentFrame().evaluateJavaScript(script.code()) + @pyqtSlot('QWebFrame*', 'QWebPage::Feature') def _on_feature_permission_requested(self, frame, feature): """Ask the user for approval for geolocation/notifications.""" diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py index 68cf1d2ba..dc0ff5580 100644 --- a/qutebrowser/utils/log.py +++ b/qutebrowser/utils/log.py @@ -95,7 +95,8 @@ LOGGER_NAMES = [ 'commands', 'signals', 'downloads', 'js', 'qt', 'rfc6266', 'ipc', 'shlexer', 'save', 'message', 'config', 'sessions', - 'webelem', 'prompt', 'network', 'sql' + 'webelem', 'prompt', 'network', 'sql', + 'greasemonkey' ] @@ -144,6 +145,7 @@ webelem = logging.getLogger('webelem') prompt = logging.getLogger('prompt') network = logging.getLogger('network') sql = logging.getLogger('sql') +greasemonkey = logging.getLogger('greasemonkey') ram_handler = None From ecdde7663ffd44579c5935c5be78d92bc9816885 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 7 Jun 2017 15:40:14 +1200 Subject: [PATCH 138/415] Add greasemonkey-reload command. Also add a signal to emit when scripts are reloaded. Had to make GreasemonkeyManager inherit from QObject to get signals to work. --- qutebrowser/browser/greasemonkey.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index d1f63a4a4..d01a8f8e5 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -29,6 +29,7 @@ import glob from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.utils import log, standarddir +from qutebrowser.commands import cmdutils # TODO: GM_ bootstrap @@ -215,10 +216,26 @@ unsafeWindow = window; }) -class GreasemonkeyManager: +class GreasemonkeyManager(QObject): + + """Manager of userscripts and a greasemonkey compatible environment. + + Signals: + scripts_reloaded: Emitted when scripts are reloaded from disk. + Any any cached or already-injected scripts should be + considered obselete. + """ + + scripts_reloaded = pyqtSignal() def __init__(self, parent=None): super().__init__(parent) + self.load_scripts() + + @cmdutils.register(name='greasemonkey-reload', + instance='greasemonkey') + def load_scripts(self): + """Re-Read greasemonkey scripts from disk.""" self._run_start = [] self._run_end = [] @@ -243,6 +260,7 @@ class GreasemonkeyManager: .format(script_path)) continue log.greasemonkey.debug("Loaded script: {}".format(script.name)) + self.scripts_reloaded.emit() def scripts_for(self, url): """Fetch scripts that are registered to run for url. From 13728387d7d875bbe7bed5687330479e2e4bdc66 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 7 Jun 2017 15:41:53 +1200 Subject: [PATCH 139/415] Greasemonkey: Fix crash on undefined metadata. --- qutebrowser/browser/greasemonkey.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index d01a8f8e5..ef8635178 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -256,9 +256,12 @@ class GreasemonkeyManager(QObject): self._run_end.append(script) else: log.greasemonkey.warning("Script {} has invalid run-at " - "defined, ignoring." + "defined, defaulting to " + "document-end" .format(script_path)) - continue + # Default as per + # https://wiki.greasespot.net/Metadata_Block#.40run-at + self._run_end.append(script) log.greasemonkey.debug("Loaded script: {}".format(script.name)) self.scripts_reloaded.emit() From 25f626a436d1f898391e19fb531984d28bb0547f Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 7 Jun 2017 15:45:12 +1200 Subject: [PATCH 140/415] Greasemonkey: Add run-at document-idle. Supposed to be after all the assets have finished loading and in page js has run. Not that we can garuntee that last bit. If a script misbehaves because a precondition isn't yet met I suggest adding a defer method to the script that adds a timer until the precondition is met. Also changed the map/filter calls to use list comprehensions to keep pylint happy. Even if it does look uglier. --- qutebrowser/browser/greasemonkey.py | 18 ++++++++++++------ qutebrowser/browser/webkit/webpage.py | 12 +++++++----- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index ef8635178..058a2d0f4 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -238,6 +238,7 @@ class GreasemonkeyManager(QObject): """Re-Read greasemonkey scripts from disk.""" self._run_start = [] self._run_end = [] + self._run_idle = [] scripts_dir = os.path.abspath(_scripts_dir()) log.greasemonkey.debug("Reading scripts from: {}".format(scripts_dir)) @@ -254,6 +255,8 @@ class GreasemonkeyManager(QObject): self._run_start.append(script) elif script.run_at == 'document-end': self._run_end.append(script) + elif script.run_at == 'document-idle': + self._run_idle.append(script) else: log.greasemonkey.warning("Script {} has invalid run-at " "defined, defaulting to " @@ -269,15 +272,18 @@ class GreasemonkeyManager(QObject): """Fetch scripts that are registered to run for url. returns a tuple of lists of scripts meant to run at (document-start, - document-end) + document-end, document-idle) """ match = functools.partial(fnmatch.fnmatch, url) tester = (lambda script: - any(map(match, script.includes())) and not - any(map(match, script.excludes()))) - return (list(filter(tester, self._run_start)), - list(filter(tester, self._run_end))) + any([match(pat) for pat in script.includes]) and + not any([match(pat) for pat in script.excludes])) + return ( + [script for script in self._run_start if tester(script)], + [script for script in self._run_end if tester(script)], + [script for script in self._run_idle if tester(script)] + ) def all_scripts(self): """Return all scripts found in the configured script directory.""" - return self._run_start + self._run_end + return self._run_start + self._run_end + self._run_idle diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 9beba6ddc..095e41fe2 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -90,6 +90,8 @@ class BrowserPage(QWebPage): functools.partial(self.inject_userjs, load='start')) self.mainFrame().initialLayoutCompleted.connect( functools.partial(self.inject_userjs, load='end')) + self.mainFrame().loadFinished.connect( + functools.partial(self.inject_userjs, load='idle')) def javaScriptPrompt(self, frame, js_msg, default): """Override javaScriptPrompt to use qutebrowser prompts.""" @@ -292,20 +294,20 @@ class BrowserPage(QWebPage): """Inject user javascripts into the page. param: The page load stage to inject the corresponding scripts - for. Support values are "start" and "end", + for. Support values are "start", "end" and "idle", corresponding to the allowed values of the `@run-at` directive in the greasemonkey metadata spec. """ greasemonkey = objreg.get('greasemonkey') url = self.currentFrame().url() - start_scripts, end_scripts = greasemonkey.scripts_for(url.toDisplayString()) - log.greasemonkey.debug('scripts: {}'.format(start_scripts if start else end_scripts)) - - toload = [] + start_scripts, end_scripts, idle_scripts = \ + greasemonkey.scripts_for(url.toDisplayString()) if load == "start": toload = start_scripts elif load == "end": toload = end_scripts + elif load == "idle": + toload = idle_scripts for script in toload: log.webview.debug('Running GM script: {}'.format(script.name)) From be9f8bd0de2d8189852bb306ebd25c010bc15c76 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 7 Jun 2017 15:49:41 +1200 Subject: [PATCH 141/415] Greasemonkey: Lift greasemonkey init app.py To prepare for multiple-backend support. --- qutebrowser/app.py | 6 +++++- qutebrowser/browser/browsertab.py | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 2ed579f61..c32a208ac 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -64,7 +64,7 @@ from qutebrowser.completion.models import miscmodels from qutebrowser.commands import cmdutils, runners, cmdexc from qutebrowser.config import config, websettings, configfiles, configinit from qutebrowser.browser import (urlmarks, adblock, history, browsertab, - downloads) + downloads, greasemonkey) from qutebrowser.browser.network import proxy from qutebrowser.browser.webkit import cookies, cache from qutebrowser.browser.webkit.network import networkmanager @@ -491,6 +491,10 @@ def _init_modules(args, crash_handler): diskcache = cache.DiskCache(standarddir.cache(), parent=qApp) objreg.register('cache', diskcache) + log.init.debug("Initializing Greasemonkey...") + gm_manager = greasemonkey.GreasemonkeyManager() + objreg.register('greasemonkey', gm_manager) + log.init.debug("Misc initialization...") macros.init() # Init backend-specific stuff diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index 866943f87..547e276db 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -31,7 +31,7 @@ from qutebrowser.keyinput import modeman from qutebrowser.config import config from qutebrowser.utils import utils, objreg, usertypes, log, qtutils from qutebrowser.misc import miscwidgets, objects -from qutebrowser.browser import mouse, hints, greasemonkey +from qutebrowser.browser import mouse, hints tab_id_gen = itertools.count(0) @@ -64,10 +64,6 @@ def init(): from qutebrowser.browser.webengine import webenginetab webenginetab.init() - log.init.debug("Initializing Greasemonkey...") - gm_manager = greasemonkey.GreasemonkeyManager() - objreg.register('greasemonkey', gm_manager) - class WebTabError(Exception): From f26377351c0fc8b2e50b583ec7eb8b195d76501c Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 7 Jun 2017 16:06:50 +1200 Subject: [PATCH 142/415] Greasemonkey: Add greasemonkey hooks for webengine. For qtwebengine 5.8+ only. This is because as of 5.8 some greasemonkey script support is in upstream. That is, qtwebenginescript(collection) parses the greasemonkey metadata block and uses @include/match/exclude to decide what sites to inject a script onto and @run-at to decide when to inject it, which saves us the trouble. Notes on doing this in <5.8 are below. Scripts are currently injected into the main "world", that is the same world as the javascript from the page. This is good because it means userscripts can modify more stuff on the page but it would be nice if we could have more isolation without sacrificing functionality. I'm still looking into why my more feature-full scripts are not having any effect on the page while running in a separate world. Userscripts are added to both the default and private profile because I that if people have scripts installed they want them to run in private mode too. We are grabbing the scripts from the greasemonkey module, as opposed to reading them directly from disk, because the module adds some GM_* functions that the scripts may expect, and because that is used for webkit anyway. I have code to support qtwebengine <5.8 but didn't because I am not happy with the timing of some of the signals that we are provided regarding page load state, and the actual load state. While the difference between document-end and document-idle isn't so bad, injecting document-start scripts when a urlChanged event is emitted results in the script being injected into the environment for the page being navigated away from. Anyway, if anyone wants this for earlier webengines I can oblige them. --- qutebrowser/browser/webengine/webenginetab.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 89ba958a7..c8d7ef670 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -69,6 +69,10 @@ def init(): download_manager.install(webenginesettings.private_profile) objreg.register('webengine-download-manager', download_manager) + greasemonkey = objreg.get('greasemonkey') + greasemonkey.scripts_reloaded.connect(inject_userscripts) + inject_userscripts() + # Mapping worlds from usertypes.JsWorld to QWebEngineScript world IDs. _JS_WORLD_MAP = { @@ -79,6 +83,42 @@ _JS_WORLD_MAP = { } +def inject_userscripts(): + """Register user javascript files with the global profiles.""" + # The greasemonkey metadata block support in qtwebengine only starts at 5.8 + # Otherwise have to handle injecting the scripts into the page at very + # early load, probs same place in view as the enableJS check. + if not qtutils.version_check('5.8'): + return + + # Since we are inserting scripts into profile.scripts they won't + # just get replaced by new gm scripts like if we were injecting them + # ourselves so we need to remove all gm scripts, while not removing + # any other stuff that might have been added. Like the one for + # stylsheets. + # Could either use a different world for gm scripts, check for gm metadata + # values (would mean no non-gm userscripts), or check the code for + # _qute_script_id + for profile in [webenginesettings.default_profile, + webenginesettings.private_profile]: + scripts = profile.scripts() + for script in scripts.toList(): + if script.worldId() == QWebEngineScript.MainWorld: + scripts.remove(script) + + # Should we be adding to private profile too? + for profile in [webenginesettings.default_profile, + webenginesettings.private_profile]: + scripts = profile.scripts() + greasemonkey = objreg.get('greasemonkey') + for script in greasemonkey.all_scripts(): + new_script = QWebEngineScript() + new_script.setWorldId(QWebEngineScript.MainWorld) + new_script.setSourceCode(script.code()) + log.greasemonkey.debug('adding script: %s', new_script.name) + scripts.insert(new_script) + + class WebEngineAction(browsertab.AbstractAction): """QtWebEngine implementations related to web actions.""" From 325c595b896ded73d324b8a2a9c178b5ef1fd604 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 27 Jul 2017 19:25:29 +1200 Subject: [PATCH 143/415] Greasemonkey: Don't strip gm metadata from scripts when loading. Since we just pass them to webenginescriptcollection on that backend and that wants to parse it itself to figure out injection point etc. --- qutebrowser/browser/greasemonkey.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 058a2d0f4..a616891d1 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -31,8 +31,6 @@ from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.utils import log, standarddir from qutebrowser.commands import cmdutils -# TODO: GM_ bootstrap - def _scripts_dir(): """Get the directory of the scripts.""" @@ -186,7 +184,7 @@ unsafeWindow = window; props, _code = matches except ValueError: props = "" - script = cls(re.findall(cls.PROPS_REGEX, props), code) + script = cls(re.findall(cls.PROPS_REGEX, props), source) script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:])) return script From 799730f6864ad8f41365542d5c393ecbd52f7155 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 27 Jul 2017 21:21:21 +1200 Subject: [PATCH 144/415] Remove GM_ and userscript variables from global scope. --- qutebrowser/browser/greasemonkey.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index a616891d1..41d32bcf1 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -201,7 +201,8 @@ unsafeWindow = window; scriptName=self.name, scriptInfo=self._meta_json(), scriptMeta=self.script_meta) - return '\n'.join([gm_bootstrap, self._code]) + return '\n'.join( + ["(function(){", gm_bootstrap, self._code, "})();"]) def _meta_json(self): return json.dumps({ From 41035cb5cab72ab3c4acfb06330eab6e7b8f25b4 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sun, 5 Nov 2017 16:36:09 +1300 Subject: [PATCH 145/415] Greasemonkey: restrict page schemes that scripts can run on Scripts shouldn't run on qute://settings or source:// etc. Whitelist from: https://wiki.greasespot.net/Include_and_exclude_rules --- qutebrowser/browser/greasemonkey.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 41d32bcf1..4a8d9bdeb 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -226,6 +226,10 @@ class GreasemonkeyManager(QObject): """ scripts_reloaded = pyqtSignal() + # https://wiki.greasespot.net/Include_and_exclude_rules#Greaseable_schemes + # Limit the schemes scripts can run on due to unreasonable levels of + # exploitability + greaseable_schemes = ['http', 'https', 'ftp', 'file'] def __init__(self, parent=None): super().__init__(parent) @@ -273,6 +277,8 @@ class GreasemonkeyManager(QObject): returns a tuple of lists of scripts meant to run at (document-start, document-end, document-idle) """ + if url.split(':', 1)[0] not in self.greaseable_schemes: + return [], [], [] match = functools.partial(fnmatch.fnmatch, url) tester = (lambda script: any([match(pat) for pat in script.includes]) and From edf737ff7d604a3d14a26eb23b0b0a94f9328230 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 4 Oct 2017 20:42:10 +1300 Subject: [PATCH 146/415] Greasemonkey: move scripts for a domain into data class. Also makes scripts that don't include a greasemonkey metadata block match any url. QWebEngine already has that behaviour. --- qutebrowser/browser/greasemonkey.py | 18 ++++++++++++++++-- qutebrowser/browser/webkit/webpage.py | 10 +++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 4a8d9bdeb..d06bc9cac 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -26,6 +26,7 @@ import fnmatch import functools import glob +import attr from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.utils import log, standarddir @@ -186,6 +187,8 @@ unsafeWindow = window; props = "" script = cls(re.findall(cls.PROPS_REGEX, props), source) script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:])) + if not props: + script.includes = ['*'] return script def code(self): @@ -215,6 +218,16 @@ unsafeWindow = window; }) +@attr.s +class MatchingScripts(object): + """All userscripts registered to run on a particular url.""" + + url = attr.ib() + start = attr.ib(default=attr.Factory(list)) + end = attr.ib(default=attr.Factory(list)) + idle = attr.ib(default=attr.Factory(list)) + + class GreasemonkeyManager(QObject): """Manager of userscripts and a greasemonkey compatible environment. @@ -278,12 +291,13 @@ class GreasemonkeyManager(QObject): document-end, document-idle) """ if url.split(':', 1)[0] not in self.greaseable_schemes: - return [], [], [] + return MatchingScripts(url, [], [], []) match = functools.partial(fnmatch.fnmatch, url) tester = (lambda script: any([match(pat) for pat in script.includes]) and not any([match(pat) for pat in script.excludes])) - return ( + return MatchingScripts( + url, [script for script in self._run_start if tester(script)], [script for script in self._run_end if tester(script)], [script for script in self._run_idle if tester(script)] diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 095e41fe2..15f662e61 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -300,14 +300,14 @@ class BrowserPage(QWebPage): """ greasemonkey = objreg.get('greasemonkey') url = self.currentFrame().url() - start_scripts, end_scripts, idle_scripts = \ - greasemonkey.scripts_for(url.toDisplayString()) + scripts = greasemonkey.scripts_for(url.toDisplayString()) + if load == "start": - toload = start_scripts + toload = scripts.start elif load == "end": - toload = end_scripts + toload = scripts.end elif load == "idle": - toload = idle_scripts + toload = scripts.idle for script in toload: log.webview.debug('Running GM script: {}'.format(script.name)) From c1b912f5670756b6a96ba378e60e83e9df64ba98 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 4 Oct 2017 21:24:16 +1300 Subject: [PATCH 147/415] Greasemonkey: move inject_userscripts into webenginesettings --- .../browser/webengine/webenginesettings.py | 33 +++++++++++++++ qutebrowser/browser/webengine/webenginetab.py | 40 +------------------ 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 4bf525c46..5ce4e1a05 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -244,6 +244,39 @@ def _init_profiles(): private_profile.setSpellCheckEnabled(True) +def inject_userscripts(): + """Register user javascript files with the global profiles.""" + # The greasemonkey metadata block support in qtwebengine only starts at 5.8 + # Otherwise have to handle injecting the scripts into the page at very + # early load, probs same place in view as the enableJS check. + if not qtutils.version_check('5.8'): + return + + # Since we are inserting scripts into profile.scripts they won't + # just get replaced by new gm scripts like if we were injecting them + # ourselves so we need to remove all gm scripts, while not removing + # any other stuff that might have been added. Like the one for + # stylsheets. + # Could either use a different world for gm scripts, check for gm metadata + # values (would mean no non-gm userscripts), or check the code for + # _qute_script_id + for profile in [default_profile, private_profile]: + scripts = profile.scripts() + for script in scripts.toList(): + if script.worldId() == QWebEngineScript.MainWorld: + scripts.remove(script) + + for profile in [default_profile, private_profile]: + scripts = profile.scripts() + greasemonkey = objreg.get('greasemonkey') + for script in greasemonkey.all_scripts(): + new_script = QWebEngineScript() + new_script.setWorldId(QWebEngineScript.MainWorld) + new_script.setSourceCode(script.code()) + log.greasemonkey.debug('adding script: %s', new_script.name()) + scripts.insert(new_script) + + def init(args): """Initialize the global QWebSettings.""" if args.enable_webengine_inspector: diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index c8d7ef670..c97aaacbe 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -70,8 +70,8 @@ def init(): objreg.register('webengine-download-manager', download_manager) greasemonkey = objreg.get('greasemonkey') - greasemonkey.scripts_reloaded.connect(inject_userscripts) - inject_userscripts() + greasemonkey.scripts_reloaded.connect(webenginesettings.inject_userscripts) + webenginesettings.inject_userscripts() # Mapping worlds from usertypes.JsWorld to QWebEngineScript world IDs. @@ -83,42 +83,6 @@ _JS_WORLD_MAP = { } -def inject_userscripts(): - """Register user javascript files with the global profiles.""" - # The greasemonkey metadata block support in qtwebengine only starts at 5.8 - # Otherwise have to handle injecting the scripts into the page at very - # early load, probs same place in view as the enableJS check. - if not qtutils.version_check('5.8'): - return - - # Since we are inserting scripts into profile.scripts they won't - # just get replaced by new gm scripts like if we were injecting them - # ourselves so we need to remove all gm scripts, while not removing - # any other stuff that might have been added. Like the one for - # stylsheets. - # Could either use a different world for gm scripts, check for gm metadata - # values (would mean no non-gm userscripts), or check the code for - # _qute_script_id - for profile in [webenginesettings.default_profile, - webenginesettings.private_profile]: - scripts = profile.scripts() - for script in scripts.toList(): - if script.worldId() == QWebEngineScript.MainWorld: - scripts.remove(script) - - # Should we be adding to private profile too? - for profile in [webenginesettings.default_profile, - webenginesettings.private_profile]: - scripts = profile.scripts() - greasemonkey = objreg.get('greasemonkey') - for script in greasemonkey.all_scripts(): - new_script = QWebEngineScript() - new_script.setWorldId(QWebEngineScript.MainWorld) - new_script.setSourceCode(script.code()) - log.greasemonkey.debug('adding script: %s', new_script.name) - scripts.insert(new_script) - - class WebEngineAction(browsertab.AbstractAction): """QtWebEngine implementations related to web actions.""" From fd5d44182ba53e6e3c5fb8c71ffe15d3c44aeeaa Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 4 Oct 2017 22:39:32 +1300 Subject: [PATCH 148/415] Greasemonkey: move GM_* template into seperate file. Also ported it to jinja rather than str.format(). Also ran the js through jslint and fixed up a few very minor things. --- qutebrowser/browser/greasemonkey.py | 123 +----------------- qutebrowser/javascript/.eslintignore | 2 + .../javascript/greasemonkey_wrapper.js | 118 +++++++++++++++++ qutebrowser/utils/jinja.py | 1 + 4 files changed, 128 insertions(+), 116 deletions(-) create mode 100644 qutebrowser/javascript/greasemonkey_wrapper.js diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index d06bc9cac..e1f0b57db 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -29,7 +29,7 @@ import glob import attr from PyQt5.QtCore import pyqtSignal, QObject -from qutebrowser.utils import log, standarddir +from qutebrowser.utils import log, standarddir, jinja from qutebrowser.commands import cmdutils @@ -41,115 +41,6 @@ def _scripts_dir(): class GreasemonkeyScript: """Container class for userscripts, parses metadata blocks.""" - GM_BOOTSTRAP_TEMPLATE = r"""var _qute_script_id = "__gm_{scriptName}"; - -function GM_log(text) {{ - console.log(text); -}} - -GM_info = (function() {{ - return {{ - 'script': {scriptInfo}, - 'scriptMetaStr': {scriptMeta}, - 'scriptWillUpdate': false, - 'version': '0.0.1', - 'scriptHandler': 'Tampermonkey' //so scripts don't expect exportFunction - }}; -}}()); - -function GM_setValue(key, value) {{ - if (localStorage !== null && - typeof key === "string" && - (typeof value === "string" || - typeof value === "number" || - typeof value == "boolean")) {{ - localStorage.setItem(_qute_script_id + key, value); - }} -}} - -function GM_getValue(key, default_) {{ - if (localStorage !== null && typeof key === "string") {{ - return localStorage.getItem(_qute_script_id + key) || default_; - }} -}} - -function GM_deleteValue(key) {{ - if (localStorage !== null && typeof key === "string") {{ - localStorage.removeItem(_qute_script_id + key); - }} -}} - -function GM_listValues() {{ - var i; - var keys = []; - for (i = 0; i < localStorage.length; ++i) {{ - if (localStorage.key(i).startsWith(_qute_script_id)) {{ - keys.push(localStorage.key(i)); - }} - }} - return keys; -}} - -function GM_openInTab(url) {{ - window.open(url); -}} - - -// Almost verbatim copy from Eric -function GM_xmlhttpRequest(/* object */ details) {{ - details.method = details.method.toUpperCase() || "GET"; - - if(!details.url) {{ - throw("GM_xmlhttpRequest requires an URL."); - }} - - // build XMLHttpRequest object - var oXhr = new XMLHttpRequest; - // run it - if("onreadystatechange" in details) - oXhr.onreadystatechange = function() {{ - details.onreadystatechange(oXhr) - }}; - if("onload" in details) - oXhr.onload = function() {{ details.onload(oXhr) }}; - if("onerror" in details) - oXhr.onerror = function() {{ details.onerror(oXhr) }}; - - oXhr.open(details.method, details.url, true); - - if("headers" in details) - for(var header in details.headers) - oXhr.setRequestHeader(header, details.headers[header]); - - if("data" in details) - oXhr.send(details.data); - else - oXhr.send(); -}} - -function GM_addStyle(/* String */ styles) {{ - var head = document.getElementsByTagName("head")[0]; - if (head === undefined) {{ - document.onreadystatechange = function() {{ - if (document.readyState == "interactive") {{ - var oStyle = document.createElement("style"); - oStyle.setAttribute("type", "text/css"); - oStyle.appendChild(document.createTextNode(styles)); - document.getElementsByTagName("head")[0].appendChild(oStyle); - }} - }} - }} - else {{ - var oStyle = document.createElement("style"); - oStyle.setAttribute("type", "text/css"); - oStyle.appendChild(document.createTextNode(styles)); - head.appendChild(oStyle); - }} -}} - -unsafeWindow = window; -""" - def __init__(self, properties, code): self._code = code self.includes = [] @@ -200,12 +91,12 @@ unsafeWindow = window; browser's debugger/inspector will not match up to the line numbers in the source script directly. """ - gm_bootstrap = self.GM_BOOTSTRAP_TEMPLATE.format( - scriptName=self.name, - scriptInfo=self._meta_json(), - scriptMeta=self.script_meta) - return '\n'.join( - ["(function(){", gm_bootstrap, self._code, "})();"]) + return jinja.js_environment.get_template( + 'greasemonkey_wrapper.js').render( + scriptName=self.name, + scriptInfo=self._meta_json(), + scriptMeta=self.script_meta, + scriptSource=self._code) def _meta_json(self): return json.dumps({ diff --git a/qutebrowser/javascript/.eslintignore b/qutebrowser/javascript/.eslintignore index ca4d3c667..036a72cfe 100644 --- a/qutebrowser/javascript/.eslintignore +++ b/qutebrowser/javascript/.eslintignore @@ -1,2 +1,4 @@ # Upstream Mozilla's code pac_utils.js +# Actually a jinja template so eslint chokes on the {{}} syntax. +greasemonkey_wrapper.js diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js new file mode 100644 index 000000000..b49dc2c02 --- /dev/null +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -0,0 +1,118 @@ +(function () { + var _qute_script_id = "__gm_{{ scriptName }}"; + + function GM_log(text) { + console.log(text); + } + + var GM_info = (function () { + return { + 'script': {{ scriptInfo }}, + 'scriptMetaStr': {{ scriptMeta }}, + 'scriptWillUpdate': false, + 'version': '0.0.1', + 'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction + }; + }()); + + function GM_setValue(key, value) { + if (localStorage !== null && + typeof key === "string" && + (typeof value === "string" || + typeof value === "number" || + typeof value === "boolean")) { + localStorage.setItem(_qute_script_id + key, value); + } + } + + function GM_getValue(key, default_) { + if (localStorage !== null && typeof key === "string") { + return localStorage.getItem(_qute_script_id + key) || default_; + } + } + + function GM_deleteValue(key) { + if (localStorage !== null && typeof key === "string") { + localStorage.removeItem(_qute_script_id + key); + } + } + + function GM_listValues() { + var i, keys = []; + for (i = 0; i < localStorage.length; i = i + 1) { + if (localStorage.key(i).startsWith(_qute_script_id)) { + keys.push(localStorage.key(i)); + } + } + return keys; + } + + function GM_openInTab(url) { + window.open(url); + } + + + // Almost verbatim copy from Eric + function GM_xmlhttpRequest(/* object */ details) { + details.method = details.method.toUpperCase() || "GET"; + + if (!details.url) { + throw ("GM_xmlhttpRequest requires an URL."); + } + + // build XMLHttpRequest object + var oXhr = new XMLHttpRequest(); + // run it + if ("onreadystatechange" in details) { + oXhr.onreadystatechange = function () { + details.onreadystatechange(oXhr); + }; + } + if ("onload" in details) { + oXhr.onload = function () { details.onload(oXhr) }; + } + if ("onerror" in details) { + oXhr.onerror = function () { details.onerror(oXhr) }; + } + + oXhr.open(details.method, details.url, true); + + if ("headers" in details) { + for (var header in details.headers) { + oXhr.setRequestHeader(header, details.headers[header]); + } + } + + if ("data" in details) { + oXhr.send(details.data); + } else { + oXhr.send(); + } + } + + function GM_addStyle(/* String */ styles) { + var head = document.getElementsByTagName("head")[0]; + if (head === undefined) { + document.onreadystatechange = function () { + if (document.readyState == "interactive") { + var oStyle = document.createElement("style"); + oStyle.setAttribute("type", "text/css"); + oStyle.appendChild(document.createTextNode(styles)); + document.getElementsByTagName("head")[0].appendChild(oStyle); + } + } + } + else { + var oStyle = document.createElement("style"); + oStyle.setAttribute("type", "text/css"); + oStyle.appendChild(document.createTextNode(styles)); + head.appendChild(oStyle); + } + } + + unsafeWindow = window; + + //====== The actual user script source ======// +{{ scriptSource }} + //====== End User Script ======// +})(); diff --git a/qutebrowser/utils/jinja.py b/qutebrowser/utils/jinja.py index e7b536b60..b6f53645b 100644 --- a/qutebrowser/utils/jinja.py +++ b/qutebrowser/utils/jinja.py @@ -136,3 +136,4 @@ def render(template, **kwargs): environment = Environment() +js_environment = jinja2.Environment(loader=Loader('javascript')) From a7f41b4564dd39247893662ebeb23170a290c85f Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 4 Oct 2017 23:25:22 +1300 Subject: [PATCH 149/415] Greasemonkey: ensure only GM scripts are cleaned up on reload. WebEngine only. Previously we were just removing every script from the main world. But some other scripts might got here in the future so new we are overriding the name field to add a GM- prefix so hopefully we only remove greasemonkey scripts before adding new ones. --- qutebrowser/browser/webengine/webenginesettings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 5ce4e1a05..18ba98fd6 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -263,7 +263,9 @@ def inject_userscripts(): for profile in [default_profile, private_profile]: scripts = profile.scripts() for script in scripts.toList(): - if script.worldId() == QWebEngineScript.MainWorld: + if script.name().startswith("GM-"): + log.greasemonkey.debug('removing script: {}' + .format(script.name())) scripts.remove(script) for profile in [default_profile, private_profile]: @@ -273,6 +275,7 @@ def inject_userscripts(): new_script = QWebEngineScript() new_script.setWorldId(QWebEngineScript.MainWorld) new_script.setSourceCode(script.code()) + new_script.setName("GM-{}".format(script.name)) log.greasemonkey.debug('adding script: %s', new_script.name()) scripts.insert(new_script) From d93c583c0d8f1859be2e306f21477ee2444b2d83 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 7 Oct 2017 17:18:48 +1300 Subject: [PATCH 150/415] Greasemonkey: Escape jinja variables for JS strings. --- qutebrowser/javascript/greasemonkey_wrapper.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index b49dc2c02..a5079b89a 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -1,5 +1,5 @@ (function () { - var _qute_script_id = "__gm_{{ scriptName }}"; + var _qute_script_id = "__gm_"+{{ scriptName | tojson }}; function GM_log(text) { console.log(text); @@ -7,8 +7,8 @@ var GM_info = (function () { return { - 'script': {{ scriptInfo }}, - 'scriptMetaStr': {{ scriptMeta }}, + 'script': {{ scriptInfo | tojson }}, + 'scriptMetaStr': {{ scriptMeta | tojson }}, 'scriptWillUpdate': false, 'version': '0.0.1', 'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction From 5e49e7eef21ef1abaf327b1c060145f67ba6f868 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 7 Oct 2017 17:22:57 +1300 Subject: [PATCH 151/415] Greasemonkey: Throw Errors if GM_ function args wrong type. These argument type restrictions are mentioned on the greasespot pages for these value storage functions. We could call JSON.dumps() instead but better to push that onto the caller so we don't have to try handle deserialization. Also removes the check for localstorage because everyone has supported that for years. --- .../javascript/greasemonkey_wrapper.js | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index a5079b89a..605f82d5a 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -16,25 +16,29 @@ }()); function GM_setValue(key, value) { - if (localStorage !== null && - typeof key === "string" && - (typeof value === "string" || - typeof value === "number" || - typeof value === "boolean")) { - localStorage.setItem(_qute_script_id + key, value); + if (typeof key !== "string") { + throw new Error("GM_setValue requires the first parameter to be of type string, not '"+typeof key+"'"); } + if (typeof value !== "string" || + typeof value !== "number" || + typeof value !== "boolean") { + throw new Error("GM_setValue requires the second parameter to be of type string, number or boolean, not '"+typeof value+"'"); + } + localStorage.setItem(_qute_script_id + key, value); } function GM_getValue(key, default_) { - if (localStorage !== null && typeof key === "string") { - return localStorage.getItem(_qute_script_id + key) || default_; + if (typeof key !== "string") { + throw new Error("GM_getValue requires the first parameter to be of type string, not '"+typeof key+"'"); } + return localStorage.getItem(_qute_script_id + key) || default_; } function GM_deleteValue(key) { - if (localStorage !== null && typeof key === "string") { - localStorage.removeItem(_qute_script_id + key); + if (typeof key !== "string") { + throw new Error("GM_deleteValue requires the first parameter to be of type string, not '"+typeof key+"'"); } + localStorage.removeItem(_qute_script_id + key); } function GM_listValues() { From d318178567f8e4236ded6ff350628127d4be7496 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 7 Oct 2017 17:32:21 +1300 Subject: [PATCH 152/415] Greasemonkey: Fix metadata block regex. This regex was broken since the original PR and subsequent code seemed to be working around it. Before re.split was returning [everything up to /UserScript, everything else], now it returns [before UserScript, metadata, after /UserScript], which is good. Also I added the check for the UserScript line starting at column 0 as per spec. --- qutebrowser/browser/greasemonkey.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index e1f0b57db..a0eb109fd 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -60,7 +60,7 @@ class GreasemonkeyScript: elif name == 'run-at': self.run_at = value - HEADER_REGEX = r'// ==UserScript==.|\n+// ==/UserScript==\n' + HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n' PROPS_REGEX = r'// @(?P[^\s]+)\s+(?P.+)' @classmethod @@ -71,13 +71,13 @@ class GreasemonkeyScript: Parses the greasemonkey metadata block, if present, to fill out attributes. """ - matches = re.split(cls.HEADER_REGEX, source, maxsplit=1) + matches = re.split(cls.HEADER_REGEX, source, maxsplit=2) try: - props, _code = matches + _, props, _code = matches except ValueError: props = "" script = cls(re.findall(cls.PROPS_REGEX, props), source) - script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:])) + script.script_meta = props if not props: script.includes = ['*'] return script From 209e43e0baa0eda4c3258abc2dad75618020db99 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 7 Oct 2017 17:33:00 +1300 Subject: [PATCH 153/415] Greasemonkey: Match against percent encoded urls only. This change requires urls specified in @include, @exclude and @matches directives in metadata blocks to be in the same form that QUrl.toEncoded() returns. That is a punycoded domain and percent encoded path and query. This seems to be what Tampermonkey on chrome expects to. Also changes the scripts_for() function to take a QUrl arg so the caller doesn't need to worry about encodings. --- qutebrowser/browser/greasemonkey.py | 5 +++-- qutebrowser/browser/webkit/webpage.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index a0eb109fd..3243ed8d8 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -181,9 +181,10 @@ class GreasemonkeyManager(QObject): returns a tuple of lists of scripts meant to run at (document-start, document-end, document-idle) """ - if url.split(':', 1)[0] not in self.greaseable_schemes: + if url.scheme() not in self.greaseable_schemes: return MatchingScripts(url, [], [], []) - match = functools.partial(fnmatch.fnmatch, url) + match = functools.partial(fnmatch.fnmatch, + str(url.toEncoded(), 'utf-8')) tester = (lambda script: any([match(pat) for pat in script.includes]) and not any([match(pat) for pat in script.excludes])) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 15f662e61..e48519189 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -300,7 +300,7 @@ class BrowserPage(QWebPage): """ greasemonkey = objreg.get('greasemonkey') url = self.currentFrame().url() - scripts = greasemonkey.scripts_for(url.toDisplayString()) + scripts = greasemonkey.scripts_for(url) if load == "start": toload = scripts.start From efde31aa5700f41b75ee7f119d6c3cacea9cdf90 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sun, 8 Oct 2017 12:35:01 +1300 Subject: [PATCH 154/415] Greasemonkey: Support QTWebEngine versions < 5.8 QTWebEngine 5.8 added support for parsing greasemonkey metadata blocks and scripts added to the QWebEngineScriptCollection of a page or its profile and then deciding what urls to run those scripts on and at what point in the load process to run them. For earlier versions we must do that work ourselves. But with the additional handicap of the less rich qtwebengine api. We have acceptNavigationRequest, loadStarted, loadProgress, loadFinished, urlChanged to choose from regarding points at which to register scripts for the current page. Adding scripts on acceptNavigation loadStarted and loadFinished causes scripts to run too early or too late (eg on the pages being navigated from/to) and not run on the desired page at the time they are inserted. We could maybe do some more sophisticated stuff with loadProgress but it didn't have any better behaviour in the brief testing I gave it. Registering scripts on the urlChanged event seems to work fine. Even if it seems like there could be problems with the signal firing too often, due to not necessarily being tied to the page load progress, that doesn't seem to have an effect in practice. The event is fired when, for example, the url fragment changes and even if we add a new script to the collection (or remove an existing one) it doesn't have an effect on what is running on the page. I suspect all of those timing issues is due to the signals being forwarded fairly directly from the underlying chomium/blink code but the webengine script stuff only being pushed back to the implementation on certain events. Anyway, using urlChanged seems to work fine due to some quirk(s) of the implementation. That might change with later development but this codepath is only ever going to be used for version 5.7. There are other potential optimizations like not removing and then re-adding scripts for the current page. But they probably wouldn't do anything anyway, or at least anything that you would expect. --- qutebrowser/browser/webengine/webview.py | 45 ++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index 56bd1eb5a..35d6fca40 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -23,12 +23,14 @@ import functools from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, PYQT_VERSION from PyQt5.QtGui import QPalette -from PyQt5.QtWebEngineWidgets import QWebEngineView, QWebEnginePage +from PyQt5.QtWebEngineWidgets import (QWebEngineView, QWebEnginePage, + QWebEngineScript) from qutebrowser.browser import shared from qutebrowser.browser.webengine import certificateerror, webenginesettings from qutebrowser.config import config -from qutebrowser.utils import log, debug, usertypes, jinja, urlutils, message +from qutebrowser.utils import (log, debug, usertypes, jinja, urlutils, message, + objreg, qtutils) class WebEngineView(QWebEngineView): @@ -135,6 +137,7 @@ class WebEnginePage(QWebEnginePage): self._theme_color = theme_color self._set_bg_color() config.instance.changed.connect(self._set_bg_color) + self.urlChanged.connect(self._inject_userjs) @config.change_filter('colors.webpage.bg') def _set_bg_color(self): @@ -300,3 +303,41 @@ class WebEnginePage(QWebEnginePage): message.error(msg) return False return True + + @pyqtSlot('QUrl') + def _inject_userjs(self, url): + """Inject userscripts registered for `url` into the current page.""" + if qtutils.version_check('5.8'): + # Handled in webenginetab with the builtin greasemonkey + # support. + return + + # Using QWebEnginePage.scripts() to hold the user scripts means + # we don't have to worry ourselves about where to inject the + # page but also means scripts hang around for the tab lifecycle. + # So clear them here. + scripts = self.scripts() + for script in scripts.toList(): + if script.name().startswith("GM-"): + really_removed = scripts.remove(script) + log.greasemonkey.debug("Removing ({}) script: {}" + .format(really_removed, script.name())) + + def _add_script(script, injection_point): + new_script = QWebEngineScript() + new_script.setInjectionPoint(injection_point) + new_script.setWorldId(QWebEngineScript.MainWorld) + new_script.setSourceCode(script.code()) + new_script.setName("GM-{}".format(script.name)) + log.greasemonkey.debug("Adding script: {}" + .format(new_script.name())) + scripts.insert(new_script) + + greasemonkey = objreg.get('greasemonkey') + matching_scripts = greasemonkey.scripts_for(url) + for script in matching_scripts.start: + _add_script(script, QWebEngineScript.DocumentCreation) + for script in matching_scripts.end: + _add_script(script, QWebEngineScript.DocumentReady) + for script in matching_scripts.idle: + _add_script(script, QWebEngineScript.Deferred) From fb019b2dab342a98ae8e364e893650fecd42fd62 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 10 Oct 2017 20:45:10 +1300 Subject: [PATCH 155/415] Address second round line comments. Add qute version to GM_info object in GM wrapper. Support using the greasemonkey @namespace metadata for its intended purpose of avoiding name collisions. Get a nice utf8 encoded string from a QUrl more better. --- qutebrowser/browser/greasemonkey.py | 9 +++++--- .../javascript/greasemonkey_wrapper.js | 22 +++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 3243ed8d8..b58a558a5 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -27,7 +27,7 @@ import functools import glob import attr -from PyQt5.QtCore import pyqtSignal, QObject +from PyQt5.QtCore import pyqtSignal, QObject, QUrl from qutebrowser.utils import log, standarddir, jinja from qutebrowser.commands import cmdutils @@ -47,10 +47,13 @@ class GreasemonkeyScript: self.excludes = [] self.description = None self.name = None + self.namespace = None self.run_at = None for name, value in properties: if name == 'name': self.name = value + elif name == 'namespace': + self.namespace = value elif name == 'description': self.description = value elif name in ['include', 'match']: @@ -93,7 +96,7 @@ class GreasemonkeyScript: """ return jinja.js_environment.get_template( 'greasemonkey_wrapper.js').render( - scriptName=self.name, + scriptName="/".join([self.namespace or '', self.name]), scriptInfo=self._meta_json(), scriptMeta=self.script_meta, scriptSource=self._code) @@ -184,7 +187,7 @@ class GreasemonkeyManager(QObject): if url.scheme() not in self.greaseable_schemes: return MatchingScripts(url, [], [], []) match = functools.partial(fnmatch.fnmatch, - str(url.toEncoded(), 'utf-8')) + url.toString(QUrl.FullyEncoded)) tester = (lambda script: any([match(pat) for pat in script.includes]) and not any([match(pat) for pat in script.excludes])) diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index 605f82d5a..eb2d8fea1 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -5,15 +5,19 @@ console.log(text); } - var GM_info = (function () { - return { - 'script': {{ scriptInfo | tojson }}, - 'scriptMetaStr': {{ scriptMeta | tojson }}, - 'scriptWillUpdate': false, - 'version': '0.0.1', - 'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction - }; - }()); + var GM_info = { + 'script': {{ scriptInfo }}, + 'scriptMetaStr': {{ scriptMeta | tojson }}, + 'scriptWillUpdate': false, + 'version': "0.0.1", + 'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction + }; + + function checkKey(key, funcName) { + if (typeof key !== "string") { + throw new Error(funcName+" requires the first parameter to be of type string, not '"+typeof key+"'"); + } + } function GM_setValue(key, value) { if (typeof key !== "string") { From c0832eb04b87b6762f4d84440039bc6cf8dc472b Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 10 Oct 2017 20:52:41 +1300 Subject: [PATCH 156/415] Greasemonkey: support @nosubframes. And run on frames by default. At least on webengine. There is probably some api to enumerate frames on a webkit page. Not tested. --- qutebrowser/browser/greasemonkey.py | 5 +++++ qutebrowser/browser/webengine/webenginesettings.py | 4 +++- qutebrowser/browser/webengine/webview.py | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index b58a558a5..cd6fce27a 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -49,6 +49,9 @@ class GreasemonkeyScript: self.name = None self.namespace = None self.run_at = None + self.script_meta = None + # Running on subframes is only supported on the qtwebengine backend. + self.runs_on_sub_frames = True for name, value in properties: if name == 'name': self.name = value @@ -62,6 +65,8 @@ class GreasemonkeyScript: self.excludes.append(value) elif name == 'run-at': self.run_at = value + elif name == 'noframes': + self.runs_on_sub_frames = False HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n' PROPS_REGEX = r'// @(?P[^\s]+)\s+(?P.+)' diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 18ba98fd6..dade671c8 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -276,7 +276,9 @@ def inject_userscripts(): new_script.setWorldId(QWebEngineScript.MainWorld) new_script.setSourceCode(script.code()) new_script.setName("GM-{}".format(script.name)) - log.greasemonkey.debug('adding script: %s', new_script.name()) + new_script.setRunsOnSubFrames(script.runs_on_sub_frames) + log.greasemonkey.debug('adding script: {}' + .format(new_script.name())) scripts.insert(new_script) diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index 35d6fca40..af4476d4a 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -329,6 +329,7 @@ class WebEnginePage(QWebEnginePage): new_script.setWorldId(QWebEngineScript.MainWorld) new_script.setSourceCode(script.code()) new_script.setName("GM-{}".format(script.name)) + new_script.setRunsOnSubFrames(script.runs_on_sub_frames) log.greasemonkey.debug("Adding script: {}" .format(new_script.name())) scripts.insert(new_script) From 7c497427ce7cdaf3d23362cdae570b9660d7cf45 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 10 Oct 2017 20:54:58 +1300 Subject: [PATCH 157/415] Greasemonkey: various javascript fixups to GM wrapper template. Thanks to @sandrosc. A few breaking changes fixed (default method to GM_xhr not working, GM_listvalues not cleaning up output, GM_setvalue param checking logic wrong) and a few hygenic changes made. --- .../javascript/greasemonkey_wrapper.js | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index eb2d8fea1..7a271375d 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -20,11 +20,9 @@ } function GM_setValue(key, value) { - if (typeof key !== "string") { - throw new Error("GM_setValue requires the first parameter to be of type string, not '"+typeof key+"'"); - } - if (typeof value !== "string" || - typeof value !== "number" || + checkKey(key, "GM_setValue"); + if (typeof value !== "string" && + typeof value !== "number" && typeof value !== "boolean") { throw new Error("GM_setValue requires the second parameter to be of type string, number or boolean, not '"+typeof value+"'"); } @@ -32,24 +30,20 @@ } function GM_getValue(key, default_) { - if (typeof key !== "string") { - throw new Error("GM_getValue requires the first parameter to be of type string, not '"+typeof key+"'"); - } + checkKey(key, "GM_getValue"); return localStorage.getItem(_qute_script_id + key) || default_; } function GM_deleteValue(key) { - if (typeof key !== "string") { - throw new Error("GM_deleteValue requires the first parameter to be of type string, not '"+typeof key+"'"); - } + checkKey(key, "GM_deleteValue"); localStorage.removeItem(_qute_script_id + key); } function GM_listValues() { - var i, keys = []; - for (i = 0; i < localStorage.length; i = i + 1) { + var keys = []; + for (var i = 0; i < localStorage.length; i++) { if (localStorage.key(i).startsWith(_qute_script_id)) { - keys.push(localStorage.key(i)); + keys.push(localStorage.key(i).slice(_qute_script_id.length)); } } return keys; @@ -62,7 +56,7 @@ // Almost verbatim copy from Eric function GM_xmlhttpRequest(/* object */ details) { - details.method = details.method.toUpperCase() || "GET"; + details.method = details.method ? details.method.toUpperCase() : "GET"; if (!details.url) { throw ("GM_xmlhttpRequest requires an URL."); @@ -99,26 +93,24 @@ } function GM_addStyle(/* String */ styles) { + var oStyle = document.createElement("style"); + oStyle.setAttribute("type", "text/css"); + oStyle.appendChild(document.createTextNode(styles)); + var head = document.getElementsByTagName("head")[0]; if (head === undefined) { document.onreadystatechange = function () { if (document.readyState == "interactive") { - var oStyle = document.createElement("style"); - oStyle.setAttribute("type", "text/css"); - oStyle.appendChild(document.createTextNode(styles)); document.getElementsByTagName("head")[0].appendChild(oStyle); } } } else { - var oStyle = document.createElement("style"); - oStyle.setAttribute("type", "text/css"); - oStyle.appendChild(document.createTextNode(styles)); head.appendChild(oStyle); } } - unsafeWindow = window; + var unsafeWindow = window; //====== The actual user script source ======// {{ scriptSource }} From 4c3461038dc4fe2a0732860f064f13a3d1c9f298 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 10 Oct 2017 20:59:27 +1300 Subject: [PATCH 158/415] Greasemonkey: add minimal end-to-end test. Just runs a greasemonkey script on a test page and uses console.log to ensure it is running. Tests @include, and basic happy path greasemonkey.py operation (loading and parsing script, scrip_for on webkit), only testing document-start injecting point but that is the troublsome one at this point. Tested on py35 debian unstable (oldwebkit and qtwebengine5.9) debian stable qtwebengine5.7. Note the extra :reload call for qt5.7 because document-start scripts don't seem to run on the first page load with the current insertion point. I need to look into this more to look at ways of fixing this. --- tests/end2end/features/javascript.feature | 9 ++++++++ tests/end2end/features/test_javascript_bdd.py | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature index a309d6187..66d108125 100644 --- a/tests/end2end/features/javascript.feature +++ b/tests/end2end/features/javascript.feature @@ -123,3 +123,12 @@ Feature: Javascript stuff And I wait for "[*/data/javascript/windowsize.html:*] loaded" in the log And I run :tab-next Then the window sizes should be the same + + Scenario: Have a greasemonkey script run on a page + When I have a greasemonkey file saved + And I run :greasemonkey-reload + And I open data/title.html + # This second reload is required in webengine < 5.8 for scripts + # registered to run at document-start, some sort of timing issue. + And I run :reload + Then the javascript message "Script is running." should be logged diff --git a/tests/end2end/features/test_javascript_bdd.py b/tests/end2end/features/test_javascript_bdd.py index 9f6c021ce..cb81e1ef6 100644 --- a/tests/end2end/features/test_javascript_bdd.py +++ b/tests/end2end/features/test_javascript_bdd.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . +import os.path + import pytest_bdd as bdd bdd.scenarios('javascript.feature') @@ -29,3 +31,23 @@ def check_window_sizes(quteproc): hidden_size = hidden.message.split()[-1] visible_size = visible.message.split()[-1] assert hidden_size == visible_size + +test_gm_script=""" +// ==UserScript== +// @name Qutebrowser test userscript +// @namespace invalid.org +// @include http://localhost:*/data/title.html +// @exclude ??? +// @run-at document-start +// ==/UserScript== +console.log("Script is running on " + window.location.pathname); +""" + +@bdd.when("I have a greasemonkey file saved") +def create_greasemonkey_file(quteproc): + script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey') + os.mkdir(script_path) + file_path = os.path.join(script_path, 'test.user.js') + with open(file_path, 'w', encoding='utf-8') as f: + f.write(test_gm_script) + From 9aeb5775c1856a612d72e4bb2e791d3a0c50775c Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 24 Oct 2017 21:19:22 +1300 Subject: [PATCH 159/415] greasemonkey: run scripts on subframes on webkit Use the `QWebPage.frameCreated` signal to get notifications of subframes and connect the javascript injection triggering signals on those frames too. I had to add a `url = url() or requestedUrl()` bit in there because the inject_userjs method was getting called to early or something when frame.url() wasn't set or was set to the previous page so we were passing the wrong url to greasemonkey.scripts_for(). I ran into a bizarre (I maybe it is completely obvious and I just don't see it) issue where the signals attached to the main frame that were connected to a partial function with the main frame as an argument were not getting emitted, or at least those partial functions were not being called. I worked around it by using None to mean defaulting to the main frame in a couple of places. --- qutebrowser/browser/greasemonkey.py | 1 - qutebrowser/browser/webkit/webpage.py | 63 +++++++++++++++++++++------ 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index cd6fce27a..c8ada7b9a 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -50,7 +50,6 @@ class GreasemonkeyScript: self.namespace = None self.run_at = None self.script_meta = None - # Running on subframes is only supported on the qtwebengine backend. self.runs_on_sub_frames = True for name, value in properties: if name == 'name': diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index e48519189..299815cc3 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -86,12 +86,33 @@ class BrowserPage(QWebPage): self.on_save_frame_state_requested) self.restoreFrameStateRequested.connect( self.on_restore_frame_state_requested) - self.mainFrame().javaScriptWindowObjectCleared.connect( - functools.partial(self.inject_userjs, load='start')) - self.mainFrame().initialLayoutCompleted.connect( - functools.partial(self.inject_userjs, load='end')) - self.mainFrame().loadFinished.connect( - functools.partial(self.inject_userjs, load='idle')) + self.connect_userjs_signals(None) + self.frameCreated.connect(self.connect_userjs_signals) + + @pyqtSlot('QWebFrame*') + def connect_userjs_signals(self, frame_arg): + """ + Connect the signals used as triggers for injecting user + javascripts into `frame_arg`. + """ + # If we pass whatever self.mainFrame() or self.currentFrame() returns + # at init time into the partial functions which the signals + # below call then the signals don't seem to be called at all for + # the main frame of the first tab. I have no idea why I am + # seeing this behavior. Replace the None in the call to this + # function in __init__ with self.mainFrame() and try for + # yourself. + if frame_arg: + frame = frame_arg + else: + frame = self.mainFrame() + + frame.javaScriptWindowObjectCleared.connect( + functools.partial(self.inject_userjs, frame_arg, load='start')) + frame.initialLayoutCompleted.connect( + functools.partial(self.inject_userjs, frame_arg, load='end')) + frame.loadFinished.connect( + functools.partial(self.inject_userjs, frame_arg, load='idle')) def javaScriptPrompt(self, frame, js_msg, default): """Override javaScriptPrompt to use qutebrowser prompts.""" @@ -290,16 +311,24 @@ class BrowserPage(QWebPage): self.error_occurred = False @pyqtSlot() - def inject_userjs(self, load): + def inject_userjs(self, frame, load): """Inject user javascripts into the page. - param: The page load stage to inject the corresponding scripts - for. Support values are "start", "end" and "idle", - corresponding to the allowed values of the `@run-at` - directive in the greasemonkey metadata spec. + Args: + frame: The QWebFrame to inject the user scripts into, or + None for the main frame. + load: The page load stage to inject the corresponding + scripts for. Support values are "start", "end" and + "idle", corresponding to the allowed values of the + `@run-at` directive in the greasemonkey metadata spec. """ + if not frame: + frame = self.mainFrame() + url = frame.url() + if url.isEmpty(): + url = frame.requestedUrl() + greasemonkey = objreg.get('greasemonkey') - url = self.currentFrame().url() scripts = greasemonkey.scripts_for(url) if load == "start": @@ -309,9 +338,15 @@ class BrowserPage(QWebPage): elif load == "idle": toload = scripts.idle + if url.isEmpty(): + # This happens during normal usage like with view source but may + # also indicate a bug. + log.greasemonkey.debug("Not running scripts for frame with no " + "url: {}".format(frame)) for script in toload: - log.webview.debug('Running GM script: {}'.format(script.name)) - self.currentFrame().evaluateJavaScript(script.code()) + if frame is self.mainFrame() or script.runs_on_sub_frames: + log.webview.debug('Running GM script: {}'.format(script.name)) + frame.evaluateJavaScript(script.code()) @pyqtSlot('QWebFrame*', 'QWebPage::Feature') def _on_feature_permission_requested(self, frame, feature): From 361a1ed6e46bf602825986e97c540b17c44b12a7 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 1 Nov 2017 23:37:53 +1300 Subject: [PATCH 160/415] Greasemonkey: change PROPS_REGEX to handle non-value keys. We weren't actually picking up the @noframes greasemonkey directive because of this. I haven't tested this very extensively but it seems to work for making the property value optional. --- qutebrowser/browser/greasemonkey.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index c8ada7b9a..845b83d85 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -68,7 +68,7 @@ class GreasemonkeyScript: self.runs_on_sub_frames = False HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n' - PROPS_REGEX = r'// @(?P[^\s]+)\s+(?P.+)' + PROPS_REGEX = r'// @(?P[^\s]+)\s*(?P.*)' @classmethod def parse(cls, source): From dd59f8d724ccd3c3e86b201511ceb07da483a53e Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 1 Nov 2017 23:41:52 +1300 Subject: [PATCH 161/415] Greasemonkey: add more end2end tests Test document-end and noframes. Because coverage.py told me to. Hopefully this doesn't slow the test run down too much, particularly the "should not be logged" bit. I'm just reusing and existing test html page that used an iframe because I'm lazy. --- tests/end2end/features/javascript.feature | 20 ++++++++--- tests/end2end/features/test_javascript_bdd.py | 35 +++++++++++++------ 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature index 66d108125..aaad84be3 100644 --- a/tests/end2end/features/javascript.feature +++ b/tests/end2end/features/javascript.feature @@ -124,11 +124,23 @@ Feature: Javascript stuff And I run :tab-next Then the window sizes should be the same - Scenario: Have a greasemonkey script run on a page - When I have a greasemonkey file saved + Scenario: Have a greasemonkey script run at page start + When I have a greasemonkey file saved for document-start with noframes unset And I run :greasemonkey-reload - And I open data/title.html + And I open data/hints/iframe.html # This second reload is required in webengine < 5.8 for scripts # registered to run at document-start, some sort of timing issue. And I run :reload - Then the javascript message "Script is running." should be logged + Then the javascript message "Script is running on /data/hints/iframe.html" should be logged + + Scenario: Have a greasemonkey script running on frames + When I have a greasemonkey file saved for document-end with noframes unset + And I run :greasemonkey-reload + And I open data/hints/iframe.html + Then the javascript message "Script is running on /data/hints/html/wrapped.html" should be logged + + Scenario: Have a greasemonkey script running on noframes + When I have a greasemonkey file saved for document-end with noframes set + And I run :greasemonkey-reload + And I open data/hints/iframe.html + Then the javascript message "Script is running on /data/hints/html/wrapped.html" should not be logged diff --git a/tests/end2end/features/test_javascript_bdd.py b/tests/end2end/features/test_javascript_bdd.py index cb81e1ef6..16896d4b5 100644 --- a/tests/end2end/features/test_javascript_bdd.py +++ b/tests/end2end/features/test_javascript_bdd.py @@ -32,22 +32,37 @@ def check_window_sizes(quteproc): visible_size = visible.message.split()[-1] assert hidden_size == visible_size -test_gm_script=""" + +test_gm_script = r""" // ==UserScript== // @name Qutebrowser test userscript // @namespace invalid.org -// @include http://localhost:*/data/title.html +// @include http://localhost:*/data/hints/iframe.html +// @include http://localhost:*/data/hints/html/wrapped.html // @exclude ??? -// @run-at document-start +// @run-at {stage} +// {frames} // ==/UserScript== console.log("Script is running on " + window.location.pathname); """ -@bdd.when("I have a greasemonkey file saved") -def create_greasemonkey_file(quteproc): - script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey') - os.mkdir(script_path) - file_path = os.path.join(script_path, 'test.user.js') - with open(file_path, 'w', encoding='utf-8') as f: - f.write(test_gm_script) +@bdd.when(bdd.parsers.parse("I have a greasemonkey file saved for {stage} " + "with noframes {frameset}")) +def create_greasemonkey_file(quteproc, stage, frameset): + script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey') + try: + os.mkdir(script_path) + except FileExistsError: + pass + file_path = os.path.join(script_path, 'test.user.js') + if frameset == "set": + frames = "@noframes" + elif frameset == "unset": + frames = "" + else: + raise ValueError("noframes can only be set or unset, " + "not {}".format(frameset)) + with open(file_path, 'w', encoding='utf-8') as f: + f.write(test_gm_script.format(stage=stage, + frames=frames)) From 92b48e77c79cf5b1206fdd953a92678315f083e6 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 1 Nov 2017 23:45:31 +1300 Subject: [PATCH 162/415] Greasemonkey: add unit tests for GreasemonkeyManager --- tests/unit/javascript/test_greasemonkey.py | 108 +++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/unit/javascript/test_greasemonkey.py diff --git a/tests/unit/javascript/test_greasemonkey.py b/tests/unit/javascript/test_greasemonkey.py new file mode 100644 index 000000000..b0ba64bdf --- /dev/null +++ b/tests/unit/javascript/test_greasemonkey.py @@ -0,0 +1,108 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: +# Copyright 2017 Florian Bruhin (The Compiler) + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Tests for qutebrowser.browser.greasemonkey.""" + +import os +import logging + +import pytest +from PyQt5.QtCore import QUrl + +from qutebrowser.browser import greasemonkey + +test_gm_script = """ +// ==UserScript== +// @name Qutebrowser test userscript +// @namespace invalid.org +// @include http://localhost:*/data/title.html +// @match http://trolol* +// @exclude https://badhost.xxx/* +// @run-at document-start +// ==/UserScript== +console.log("Script is running."); +""" + +pytestmark = pytest.mark.usefixtures('data_tmpdir') + + +def save_script(script_text, filename): + script_path = greasemonkey._scripts_dir() + try: + os.mkdir(script_path) + except FileExistsError: + pass + file_path = os.path.join(script_path, filename) + with open(file_path, 'w', encoding='utf-8') as f: + f.write(script_text) + + +def test_all(): + """Test that a script gets read from file, parsed and returned.""" + save_script(test_gm_script, 'test.user.js') + + gm_manager = greasemonkey.GreasemonkeyManager() + assert (gm_manager.all_scripts()[0].name == + "Qutebrowser test userscript") + + +@pytest.mark.parametrize("url, expected_matches", [ + # included + ('http://trololololololo.com/', 1), + # neither included nor excluded + ('http://aaaaaaaaaa.com/', 0), + # excluded + ('https://badhost.xxx/', 0), +]) +def test_get_scripts_by_url(url, expected_matches): + """Check greasemonkey include/exclude rules work.""" + save_script(test_gm_script, 'test.user.js') + gm_manager = greasemonkey.GreasemonkeyManager() + + scripts = gm_manager.scripts_for(QUrl(url)) + assert (len(scripts.start + scripts.end + scripts.idle) == + expected_matches) + + +def test_no_metadata(caplog): + """Run on all sites at document-end is the default.""" + save_script("var nothing = true;\n", 'nothing.user.js') + + with caplog.at_level(logging.WARNING): + gm_manager = greasemonkey.GreasemonkeyManager() + + scripts = gm_manager.scripts_for(QUrl('http://notamatch.invalid/')) + assert len(scripts.start + scripts.end + scripts.idle) == 1 + assert len(scripts.end) == 1 + + +def test_bad_scheme(caplog): + """qute:// isn't in the list of allowed schemes.""" + save_script("var nothing = true;\n", 'nothing.user.js') + + with caplog.at_level(logging.WARNING): + gm_manager = greasemonkey.GreasemonkeyManager() + + scripts = gm_manager.scripts_for(QUrl('qute://settings')) + assert len(scripts.start + scripts.end + scripts.idle) == 0 + + +def test_load_emits_signal(qtbot): + gm_manager = greasemonkey.GreasemonkeyManager() + with qtbot.wait_signal(gm_manager.scripts_reloaded): + gm_manager.load_scripts() From 8a5b42ffbd6ad878fa5810819086b1db77fe8842 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 4 Nov 2017 16:11:58 +1300 Subject: [PATCH 163/415] Greasemonkey: es6ify the greasemonkey wrapper js. Because backwards compatibility sucks I guess. --- .../javascript/greasemonkey_wrapper.js | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index 7a271375d..e86991040 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -1,11 +1,11 @@ (function () { - var _qute_script_id = "__gm_"+{{ scriptName | tojson }}; + const _qute_script_id = "__gm_"+{{ scriptName | tojson }}; function GM_log(text) { console.log(text); } - var GM_info = { + const GM_info = { 'script': {{ scriptInfo }}, 'scriptMetaStr': {{ scriptMeta | tojson }}, 'scriptWillUpdate': false, @@ -15,7 +15,7 @@ function checkKey(key, funcName) { if (typeof key !== "string") { - throw new Error(funcName+" requires the first parameter to be of type string, not '"+typeof key+"'"); + throw new Error(`${funcName} requires the first parameter to be of type string, not '${typeof key}'`); } } @@ -24,7 +24,7 @@ if (typeof value !== "string" && typeof value !== "number" && typeof value !== "boolean") { - throw new Error("GM_setValue requires the second parameter to be of type string, number or boolean, not '"+typeof value+"'"); + throw new Error(`GM_setValue requires the second parameter to be of type string, number or boolean, not '${typeof value}'`); } localStorage.setItem(_qute_script_id + key, value); } @@ -40,8 +40,8 @@ } function GM_listValues() { - var keys = []; - for (var i = 0; i < localStorage.length; i++) { + let keys = []; + for (let i = 0; i < localStorage.length; i++) { if (localStorage.key(i).startsWith(_qute_script_id)) { keys.push(localStorage.key(i).slice(_qute_script_id.length)); } @@ -63,7 +63,7 @@ } // build XMLHttpRequest object - var oXhr = new XMLHttpRequest(); + let oXhr = new XMLHttpRequest(); // run it if ("onreadystatechange" in details) { oXhr.onreadystatechange = function () { @@ -80,7 +80,7 @@ oXhr.open(details.method, details.url, true); if ("headers" in details) { - for (var header in details.headers) { + for (let header in details.headers) { oXhr.setRequestHeader(header, details.headers[header]); } } @@ -93,11 +93,11 @@ } function GM_addStyle(/* String */ styles) { - var oStyle = document.createElement("style"); + let oStyle = document.createElement("style"); oStyle.setAttribute("type", "text/css"); oStyle.appendChild(document.createTextNode(styles)); - var head = document.getElementsByTagName("head")[0]; + let head = document.getElementsByTagName("head")[0]; if (head === undefined) { document.onreadystatechange = function () { if (document.readyState == "interactive") { @@ -110,7 +110,7 @@ } } - var unsafeWindow = window; + const unsafeWindow = window; //====== The actual user script source ======// {{ scriptSource }} From df624944f99a0dca33900ddaed2cd49f73880ac6 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 8 Nov 2017 21:03:58 +1300 Subject: [PATCH 164/415] Greasemonkey: webkit: injected all scripts on loadFinished. The signal we were using to inject greasemonkey scripts registered to run at document-start (javaScriptWindowObjectCleared) was unreliable to non-existant. The initialLayoutCompleted signal is a bit of an odd duck too I suppose. Anyway, we don't anticipate any scripts would break from being injected when the page is finished loaded that wouldn't already have been flaky due to the complexities of the modern web. If there is an issue hopefully someone raises an issue and we can look into it. --- qutebrowser/browser/webkit/webpage.py | 51 +++++++-------------------- 1 file changed, 13 insertions(+), 38 deletions(-) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 299815cc3..f9bd51194 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -86,33 +86,18 @@ class BrowserPage(QWebPage): self.on_save_frame_state_requested) self.restoreFrameStateRequested.connect( self.on_restore_frame_state_requested) - self.connect_userjs_signals(None) + self.connect_userjs_signals(self.mainFrame()) self.frameCreated.connect(self.connect_userjs_signals) @pyqtSlot('QWebFrame*') - def connect_userjs_signals(self, frame_arg): - """ - Connect the signals used as triggers for injecting user - javascripts into `frame_arg`. - """ - # If we pass whatever self.mainFrame() or self.currentFrame() returns - # at init time into the partial functions which the signals - # below call then the signals don't seem to be called at all for - # the main frame of the first tab. I have no idea why I am - # seeing this behavior. Replace the None in the call to this - # function in __init__ with self.mainFrame() and try for - # yourself. - if frame_arg: - frame = frame_arg - else: - frame = self.mainFrame() + def connect_userjs_signals(self, frame): + """Connect userjs related signals to `frame`. - frame.javaScriptWindowObjectCleared.connect( - functools.partial(self.inject_userjs, frame_arg, load='start')) - frame.initialLayoutCompleted.connect( - functools.partial(self.inject_userjs, frame_arg, load='end')) + Connect the signals used as triggers for injecting user + javascripts into the passed QWebFrame. + """ frame.loadFinished.connect( - functools.partial(self.inject_userjs, frame_arg, load='idle')) + functools.partial(self.inject_userjs, frame)) def javaScriptPrompt(self, frame, js_msg, default): """Override javaScriptPrompt to use qutebrowser prompts.""" @@ -311,32 +296,22 @@ class BrowserPage(QWebPage): self.error_occurred = False @pyqtSlot() - def inject_userjs(self, frame, load): + def inject_userjs(self, frame): """Inject user javascripts into the page. Args: - frame: The QWebFrame to inject the user scripts into, or - None for the main frame. - load: The page load stage to inject the corresponding - scripts for. Support values are "start", "end" and - "idle", corresponding to the allowed values of the - `@run-at` directive in the greasemonkey metadata spec. + frame: The QWebFrame to inject the user scripts into. """ - if not frame: - frame = self.mainFrame() url = frame.url() if url.isEmpty(): url = frame.requestedUrl() greasemonkey = objreg.get('greasemonkey') scripts = greasemonkey.scripts_for(url) - - if load == "start": - toload = scripts.start - elif load == "end": - toload = scripts.end - elif load == "idle": - toload = scripts.idle + # QtWebKit has trouble providing us with signals representing + # page load progress at reasonable times, so we just load all + # scripts on the same event. + toload = scripts.start + scripts.end + scripts.idle if url.isEmpty(): # This happens during normal usage like with view source but may From 6933bc05b4ba93ef6d8d72ec582362a7db36aef5 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 20 Nov 2017 07:31:01 +0100 Subject: [PATCH 165/415] Add some debug logging for GreaseMonkey with QtWebKit --- qutebrowser/browser/webkit/webpage.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index f9bd51194..7c67ab64b 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -96,6 +96,8 @@ class BrowserPage(QWebPage): Connect the signals used as triggers for injecting user javascripts into the passed QWebFrame. """ + log.greasemonkey.debug("Connecting to frame {} ({})" + .format(frame, frame.url().toDisplayString())) frame.loadFinished.connect( functools.partial(self.inject_userjs, frame)) @@ -306,6 +308,9 @@ class BrowserPage(QWebPage): if url.isEmpty(): url = frame.requestedUrl() + log.greasemonkey.debug("inject_userjs called for {} ({})" + .format(frame, url.toDisplayString())) + greasemonkey = objreg.get('greasemonkey') scripts = greasemonkey.scripts_for(url) # QtWebKit has trouble providing us with signals representing From db353c40300ef8d4066026b5650c27daa2da61e3 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 20 Nov 2017 07:31:13 +0100 Subject: [PATCH 166/415] Connect the page signal for GreaseMonkey Looks like we don't get the mainFrame's loadFinished signal properly. --- qutebrowser/browser/webkit/webpage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 7c67ab64b..24a213a42 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -86,7 +86,8 @@ class BrowserPage(QWebPage): self.on_save_frame_state_requested) self.restoreFrameStateRequested.connect( self.on_restore_frame_state_requested) - self.connect_userjs_signals(self.mainFrame()) + self.loadFinished.connect( + functools.partial(self.inject_userjs, self.mainFrame())) self.frameCreated.connect(self.connect_userjs_signals) @pyqtSlot('QWebFrame*') From 0e80be2d30ac90b82db968d077d498ad9c1e8f97 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 20 Nov 2017 08:31:10 +0100 Subject: [PATCH 167/415] Clear end2end test data again after initializing If we don't do this, earlier tests can affect later ones when e.g. using "... should not be logged", as we don't really wait until a test has been fully finished. --- tests/end2end/fixtures/quteprocess.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index 27c347ca4..3164f5fde 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -527,6 +527,7 @@ class QuteProc(testprocess.Process): super().before_test() self.send_cmd(':config-clear') self._init_settings() + self.clear_data() def _init_settings(self): """Adjust some qutebrowser settings after starting.""" From 0381d74e9aac2d045567db13ffc2ccca55d57160 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Mon, 27 Nov 2017 20:06:29 +1300 Subject: [PATCH 168/415] Greasemonkey: privatise some utility functions --- qutebrowser/browser/webkit/webpage.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 24a213a42..89b293869 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -87,11 +87,11 @@ class BrowserPage(QWebPage): self.restoreFrameStateRequested.connect( self.on_restore_frame_state_requested) self.loadFinished.connect( - functools.partial(self.inject_userjs, self.mainFrame())) - self.frameCreated.connect(self.connect_userjs_signals) + functools.partial(self._inject_userjs, self.mainFrame())) + self.frameCreated.connect(self._connect_userjs_signals) @pyqtSlot('QWebFrame*') - def connect_userjs_signals(self, frame): + def _connect_userjs_signals(self, frame): """Connect userjs related signals to `frame`. Connect the signals used as triggers for injecting user @@ -100,7 +100,7 @@ class BrowserPage(QWebPage): log.greasemonkey.debug("Connecting to frame {} ({})" .format(frame, frame.url().toDisplayString())) frame.loadFinished.connect( - functools.partial(self.inject_userjs, frame)) + functools.partial(self._inject_userjs, frame)) def javaScriptPrompt(self, frame, js_msg, default): """Override javaScriptPrompt to use qutebrowser prompts.""" @@ -298,8 +298,7 @@ class BrowserPage(QWebPage): else: self.error_occurred = False - @pyqtSlot() - def inject_userjs(self, frame): + def _inject_userjs(self, frame): """Inject user javascripts into the page. Args: @@ -309,7 +308,7 @@ class BrowserPage(QWebPage): if url.isEmpty(): url = frame.requestedUrl() - log.greasemonkey.debug("inject_userjs called for {} ({})" + log.greasemonkey.debug("_inject_userjs called for {} ({})" .format(frame, url.toDisplayString())) greasemonkey = objreg.get('greasemonkey') From a4b96c3443265e58163cd06f6f18031b6585cd88 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 Nov 2017 16:02:15 +0100 Subject: [PATCH 169/415] Update check-manifest from 0.35 to 0.36 --- misc/requirements/requirements-check-manifest.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-check-manifest.txt b/misc/requirements/requirements-check-manifest.txt index 42724e0c7..954e3a562 100644 --- a/misc/requirements/requirements-check-manifest.txt +++ b/misc/requirements/requirements-check-manifest.txt @@ -1,3 +1,3 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -check-manifest==0.35 +check-manifest==0.36 From 6c47b5d2d7e0d5dd0757ecb4890bd5408515df1b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 Nov 2017 16:02:17 +0100 Subject: [PATCH 170/415] Update setuptools from 36.8.0 to 38.2.1 --- misc/requirements/requirements-pip.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pip.txt b/misc/requirements/requirements-pip.txt index cbecd84d0..4f4b0986c 100644 --- a/misc/requirements/requirements-pip.txt +++ b/misc/requirements/requirements-pip.txt @@ -3,6 +3,6 @@ appdirs==1.4.3 packaging==16.8 pyparsing==2.2.0 -setuptools==36.8.0 +setuptools==38.2.1 six==1.11.0 wheel==0.30.0 From 5df2025745d7bbdd1014bc9cb79ad675a4db590d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 Nov 2017 16:02:18 +0100 Subject: [PATCH 171/415] Update cheroot from 5.9.1 to 5.10.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 4f8da07f4..9c9ac04ad 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -2,7 +2,7 @@ attrs==17.3.0 beautifulsoup4==4.6.0 -cheroot==5.9.1 +cheroot==5.10.0 click==6.7 # colorama==0.3.9 coverage==4.4.2 From 54db781b971c1a672042a274f3b2b7f1a80b29a8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 Nov 2017 16:02:20 +0100 Subject: [PATCH 172/415] Update hunter from 2.0.1 to 2.0.2 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 9c9ac04ad..47388ec42 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -10,7 +10,7 @@ EasyProcess==0.2.3 fields==5.0.0 Flask==0.12.2 glob2==0.6 -hunter==2.0.1 +hunter==2.0.2 hypothesis==3.38.0 itsdangerous==0.24 # Jinja2==2.9.6 From 5f3001690110d8680d1311d4f6798dee9adc927a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 Nov 2017 16:02:21 +0100 Subject: [PATCH 173/415] Update hypothesis from 3.38.0 to 3.38.5 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 47388ec42..e04b70dad 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -11,7 +11,7 @@ fields==5.0.0 Flask==0.12.2 glob2==0.6 hunter==2.0.2 -hypothesis==3.38.0 +hypothesis==3.38.5 itsdangerous==0.24 # Jinja2==2.9.6 Mako==1.0.7 From b820b9d53016fa9ccb84f284c506ef75da638ccd Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 Nov 2017 16:02:22 +0100 Subject: [PATCH 174/415] Update pytest-qt from 2.2.1 to 2.3.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index e04b70dad..358dd72a6 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -28,7 +28,7 @@ pytest-cov==2.5.1 pytest-faulthandler==1.3.1 pytest-instafail==0.3.0 pytest-mock==1.6.3 -pytest-qt==2.2.1 +pytest-qt==2.3.0 pytest-repeat==0.4.1 pytest-rerunfailures==3.1 pytest-travis-fold==1.2.0 From 9a8bac18ae523af4d8710332d1231c528a8e63d2 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 27 Nov 2017 16:02:24 +0100 Subject: [PATCH 175/415] Update pluggy from 0.5.2 to 0.6.0 --- misc/requirements/requirements-tox.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tox.txt b/misc/requirements/requirements-tox.txt index 339e1de07..1308c8afd 100644 --- a/misc/requirements/requirements-tox.txt +++ b/misc/requirements/requirements-tox.txt @@ -1,6 +1,6 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -pluggy==0.5.2 +pluggy==0.6.0 py==1.5.2 tox==2.9.1 virtualenv==15.1.0 From 40e4e73e361c70bf5285ff3eea32ca3d41e2c8d4 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 26 Sep 2017 22:26:28 -0400 Subject: [PATCH 176/415] Ensure HistoryCategory works in isolation. While QSortFilterProxyModel emits layoutChanged when changing the pattern, QSqlQueryModel emits modelReset. As only layoutChanged was connected, a HistoryCategory would only work in a model that also had at least one ListCategory. The simplest solution is to have the parent model emit the signal directly. This also emits a single signal on a pattern change rather that one for each child model. Resolves #3016. --- qutebrowser/completion/models/completionmodel.py | 6 ++++-- tests/unit/completion/test_completionmodel.py | 12 ++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 398673200..004848cfb 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -60,8 +60,6 @@ class CompletionModel(QAbstractItemModel): def add_category(self, cat): """Add a completion category to the model.""" self._categories.append(cat) - cat.layoutAboutToBeChanged.connect(self.layoutAboutToBeChanged) - cat.layoutChanged.connect(self.layoutChanged) def data(self, index, role=Qt.DisplayRole): """Return the item data for index. @@ -179,8 +177,12 @@ class CompletionModel(QAbstractItemModel): pattern: The filter pattern to set. """ log.completion.debug("Setting completion pattern '{}'".format(pattern)) + # layoutChanged is broken in pyqt-5.7.1, so we must use metaObject + # https://www.riverbankcomputing.com/pipermail/pyqt/2017-January/038483.html + self.metaObject().invokeMethod(self, "layoutAboutToBeChanged") for cat in self._categories: cat.set_pattern(pattern) + self.metaObject().invokeMethod(self, "layoutChanged") def first_item(self): """Return the index of the first child (non-category) in the model.""" diff --git a/tests/unit/completion/test_completionmodel.py b/tests/unit/completion/test_completionmodel.py index 292349730..1e88838f3 100644 --- a/tests/unit/completion/test_completionmodel.py +++ b/tests/unit/completion/test_completionmodel.py @@ -68,17 +68,17 @@ def test_count(counts): assert model.count() == sum(counts) -@hypothesis.given(strategies.text()) -def test_set_pattern(pat): +@hypothesis.given(pat=strategies.text()) +def test_set_pattern(pat, qtbot): """Validate the filtering and sorting results of set_pattern.""" model = completionmodel.CompletionModel() - cats = [mock.Mock(spec=['set_pattern', 'layoutChanged', - 'layoutAboutToBeChanged']) - for _ in range(3)] + cats = [mock.Mock(spec=['set_pattern']) for _ in range(3)] for c in cats: c.set_pattern = mock.Mock(spec=[]) model.add_category(c) - model.set_pattern(pat) + with qtbot.waitSignals([model.layoutAboutToBeChanged, model.layoutChanged], + order='strict'): + model.set_pattern(pat) for c in cats: c.set_pattern.assert_called_with(pat) From 814a88b741eac871b64eb8acb712651b53d06958 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 06:24:45 +0100 Subject: [PATCH 177/415] Add nEXT to list of alternatives [ci skip] --- README.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/README.asciidoc b/README.asciidoc index a87165366..8ba51ca98 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -174,6 +174,7 @@ Active * https://luakit.github.io/luakit/[luakit] (C/Lua, GTK+ with WebKit2) * http://surf.suckless.org/[surf] (C, GTK+ with WebKit1/WebKit2) * http://www.uzbl.org/[uzbl] (C, GTK+ with WebKit1/WebKit2) +* https://github.com/nEXT-Browser/nEXT[nEXT] (Lisp, emacs-like, currently macOS only) * Chrome/Chromium addons: https://github.com/1995eaton/chromium-vim[cVim], http://vimium.github.io/[Vimium], From 4fed8518e1d7212b4a63899572ee9de24b785c93 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 06:53:41 +0100 Subject: [PATCH 178/415] Handle "out of memory" error in sql.init() --- qutebrowser/misc/sql.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index f19de698f..bcc5bac82 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -72,7 +72,18 @@ class SqliteError(SqlError): '8', # SQLITE_READONLY '13', # SQLITE_FULL ] - self.environmental = error.nativeErrorCode() in environmental_errors + # At least in init(), we can get errors like this: + # type: ConnectionError + # database text: out of memory + # driver text: Error opening database + # error code: -1 + environmental_strings = [ + "out of memory", + ] + errcode = error.nativeErrorCode() + self.environmental = ( + errcode in environmental_errors or + (errcode == -1 and error.databaseText() in environmental_strings)) def text(self): return self.error.databaseText() From 73587b1e164e295e02314b24d3682db44b210099 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 06:55:55 +0100 Subject: [PATCH 179/415] Add SQLITE_CORRUPT to environmental SQL errors --- qutebrowser/misc/sql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qutebrowser/misc/sql.py b/qutebrowser/misc/sql.py index bcc5bac82..00527e7fe 100644 --- a/qutebrowser/misc/sql.py +++ b/qutebrowser/misc/sql.py @@ -70,6 +70,7 @@ class SqliteError(SqlError): environmental_errors = [ '5', # SQLITE_BUSY ("database is locked") '8', # SQLITE_READONLY + '11', # SQLITE_CORRUPT '13', # SQLITE_FULL ] # At least in init(), we can get errors like this: From ef1825efb0f53dc7548ed38e1698cc4393a1681b Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 07:02:33 +0100 Subject: [PATCH 180/415] Handle sqlite errors during :history-clear --- qutebrowser/browser/history.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/qutebrowser/browser/history.py b/qutebrowser/browser/history.py index de9e9bb4f..1901548bf 100644 --- a/qutebrowser/browser/history.py +++ b/qutebrowser/browser/history.py @@ -21,6 +21,7 @@ import os import time +import contextlib from PyQt5.QtCore import pyqtSlot, QUrl, QTimer @@ -87,6 +88,16 @@ class WebHistory(sql.SqlTable): def __contains__(self, url): return self._contains_query.run(val=url).value() + @contextlib.contextmanager + def _handle_sql_errors(self): + try: + yield + except sql.SqlError as e: + if e.environmental: + message.error("Failed to write history: {}".format(e.text())) + else: + raise + def _rebuild_completion(self): data = {'url': [], 'title': [], 'last_atime': []} # select the latest entry for each url @@ -142,8 +153,9 @@ class WebHistory(sql.SqlTable): "history?") def _do_clear(self): - self.delete_all() - self.completion.delete_all() + with self._handle_sql_errors(): + self.delete_all() + self.completion.delete_all() def delete_url(self, url): """Remove all history entries with the given url. @@ -191,7 +203,7 @@ class WebHistory(sql.SqlTable): atime = int(atime) if (atime is not None) else int(time.time()) - try: + with self._handle_sql_errors(): self.insert({'url': self._format_url(url), 'title': title, 'atime': atime, @@ -202,11 +214,6 @@ class WebHistory(sql.SqlTable): 'title': title, 'last_atime': atime }, replace=True) - except sql.SqlError as e: - if e.environmental: - message.error("Failed to write history: {}".format(e.text())) - else: - raise def _parse_entry(self, line): """Parse a history line like '12345 http://example.com title'.""" From 1d17e7ab0337278c3b3853b8565c521b649d3a10 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 07:04:57 +0100 Subject: [PATCH 181/415] Update changelog --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index e06ddb6e3..c559c8ca2 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -130,6 +130,7 @@ Fixed - Trying to bind an empty command now doesn't crash anymore. - Fixed crash when `:config-write-py` fails to write to the given path. - Fixed crash for some users when selecting a file with Qt 5.9.3 +- Improved handling for various SQL errors v1.0.3 ------ From 56759cca6b73e6f93dbed28d775fa276f2fd7fae Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 08:16:07 +0100 Subject: [PATCH 182/415] Make more unit tests run with Qt 5.7.1 on a newer Linux --- pytest.ini | 3 ++- tests/conftest.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index edc0b2d8f..4fefa0f77 100644 --- a/pytest.ini +++ b/pytest.ini @@ -51,7 +51,8 @@ qt_log_ignore = ^Error when parsing the netrc file ^Image of format '' blocked because it is not considered safe. If you are sure it is safe to do so, you can white-list the format by setting the environment variable QTWEBKIT_IMAGEFORMAT_WHITELIST= ^QPainter::end: Painter ended with \d+ saved states - ^QSslSocket: cannot resolve * + ^QSslSocket: cannot resolve .* + ^QSslSocket: cannot call unresolved function .* ^Incompatible version of OpenSSL ^QQuickWidget::invalidateRenderControl could not make context current ^libpng warning: iCCP: known incorrect sRGB profile diff --git a/tests/conftest.py b/tests/conftest.py index da0bded55..209cb5898 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,7 @@ import warnings import sip import pytest import hypothesis -from PyQt5.QtCore import PYQT_VERSION +from PyQt5.QtCore import qVersion, PYQT_VERSION pytest.register_assert_rewrite('helpers') @@ -151,6 +151,17 @@ def pytest_ignore_collect(path): return rel_path == os.path.join('end2end', 'features') and skip_bdd +@pytest.fixture(scope='session') +def qapp_args(): + """Make QtWebEngine unit tests run on Qt 5.7.1. + + See https://github.com/qutebrowser/qutebrowser/issues/3163 + """ + if qVersion() == '5.7.1': + return [sys.argv[0], '--disable-seccomp-filter-sandbox'] + return [] + + @pytest.fixture(scope='session') def qapp(qapp): """Change the name of the QApplication instance.""" From 9a8d68aa2505a1481a0b5f5685baf6c42322f6ab Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 09:19:37 +0100 Subject: [PATCH 183/415] Mark :undo test as flaky I don't get why it fails sometimes - so another @flaky stop-gap for now. --- tests/end2end/features/tabs.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end2end/features/tabs.feature b/tests/end2end/features/tabs.feature index f28ae62ec..b38d87a6e 100644 --- a/tests/end2end/features/tabs.feature +++ b/tests/end2end/features/tabs.feature @@ -761,6 +761,7 @@ Feature: Tab management - data/numbers/3.txt - data/numbers/2.txt + @flaky Scenario: Undo a tab closed after new tab opened When I open data/numbers/1.txt And I open data/numbers/2.txt in a new tab From b407f4ab417e57aed696cece5aec2cd16517e907 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 10:53:01 +0100 Subject: [PATCH 184/415] Release v1.0.4 --- doc/changelog.asciidoc | 6 +++--- qutebrowser/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index c559c8ca2..722001ef8 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -120,8 +120,8 @@ Removed - The `x[xtb]` default bindings got removed again as many users accidentally triggered them. -v1.0.4 (unreleased) -------------------- +v1.0.4 +------ Fixed ~~~~~ @@ -131,6 +131,7 @@ Fixed - Fixed crash when `:config-write-py` fails to write to the given path. - Fixed crash for some users when selecting a file with Qt 5.9.3 - Improved handling for various SQL errors +- Fix crash when setting content.cache.size to a big value (> 2 GB) v1.0.3 ------ @@ -153,7 +154,6 @@ Fixed window - Unbinding a default keybinding twice now doesn't bind it again - Completions are now sorted correctly again when filtered -- Fix crash when setting content.cache.size to a big value (> 2 GB) v1.0.2 ------ diff --git a/qutebrowser/__init__.py b/qutebrowser/__init__.py index 35385b03a..6859abedc 100644 --- a/qutebrowser/__init__.py +++ b/qutebrowser/__init__.py @@ -26,7 +26,7 @@ __copyright__ = "Copyright 2014-2017 Florian Bruhin (The Compiler)" __license__ = "GPL" __maintainer__ = __author__ __email__ = "mail@qutebrowser.org" -__version_info__ = (1, 0, 3) +__version_info__ = (1, 0, 4) __version__ = '.'.join(str(e) for e in __version_info__) __description__ = "A keyboard-driven, vim-like browser based on PyQt5." From b6dcd4d3872f6706a899af92f35690a7ac4ed5a7 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 28 Nov 2017 07:20:49 -0500 Subject: [PATCH 185/415] Reapply "Hide quickmark/bookmark completion if empty." This reverts commit e72e8b8556561c74eb5700c7e800aa0838f30a4c. Now that the SQL category works in isolation, it is possible to hide quickmarks/bookmarks when those categories are empty. Fixes #960 --- qutebrowser/completion/models/urlmodel.py | 15 +++++--- tests/unit/completion/test_models.py | 46 ++++++++++++++++++++++- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/qutebrowser/completion/models/urlmodel.py b/qutebrowser/completion/models/urlmodel.py index 617fa74b5..1d7a075eb 100644 --- a/qutebrowser/completion/models/urlmodel.py +++ b/qutebrowser/completion/models/urlmodel.py @@ -56,14 +56,17 @@ def url(*, info): """ model = completionmodel.CompletionModel(column_widths=(40, 50, 10)) - quickmarks = ((url, name) for (name, url) - in objreg.get('quickmark-manager').marks.items()) + quickmarks = [(url, name) for (name, url) + in objreg.get('quickmark-manager').marks.items()] bookmarks = objreg.get('bookmark-manager').marks.items() - model.add_category(listcategory.ListCategory( - 'Quickmarks', quickmarks, delete_func=_delete_quickmark, sort=False)) - model.add_category(listcategory.ListCategory( - 'Bookmarks', bookmarks, delete_func=_delete_bookmark, sort=False)) + if quickmarks: + model.add_category(listcategory.ListCategory( + 'Quickmarks', quickmarks, delete_func=_delete_quickmark, + sort=False)) + if bookmarks: + model.add_category(listcategory.ListCategory( + 'Bookmarks', bookmarks, delete_func=_delete_bookmark, sort=False)) if info.config.get('completion.web_history_max_items') != 0: hist_cat = histcategory.HistoryCategory(delete_func=_delete_history) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 5276ffd2a..1481ec47a 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -356,6 +356,50 @@ def test_url_completion(qtmodeltester, web_history_populated, }) +def test_url_completion_no_quickmarks(qtmodeltester, web_history_populated, + quickmark_manager_stub, bookmarks, info): + """Test that the quickmark category is gone with no quickmarks.""" + model = urlmodel.url(info=info) + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + _check_completions(model, { + "Bookmarks": [ + ('https://github.com', 'GitHub', None), + ('https://python.org', 'Welcome to Python.org', None), + ('http://qutebrowser.org', 'qutebrowser | qutebrowser', None), + ], + "History": [ + ('https://github.com', 'https://github.com', '2016-05-01'), + ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ], + }) + + +def test_url_completion_no_bookmarks(qtmodeltester, web_history_populated, + quickmarks, bookmark_manager_stub, info): + """Test that the bookmarks category is gone with no bookmarks.""" + model = urlmodel.url(info=info) + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + _check_completions(model, { + "Quickmarks": [ + ('https://wiki.archlinux.org', 'aw', None), + ('https://wikipedia.org', 'wiki', None), + ('https://duckduckgo.com', 'ddg', None), + ], + "History": [ + ('https://github.com', 'https://github.com', '2016-05-01'), + ('https://python.org', 'Welcome to Python.org', '2016-03-08'), + ('http://qutebrowser.org', 'qutebrowser', '2015-09-05'), + ], + }) + + @pytest.mark.parametrize('url, title, pattern, rowcount', [ ('example.com', 'Site Title', '', 1), ('example.com', 'Site Title', 'ex', 1), @@ -382,7 +426,7 @@ def test_url_completion_pattern(web_history, quickmark_manager_stub, model = urlmodel.url(info=info) model.set_pattern(pattern) # 2, 0 is History - assert model.rowCount(model.index(2, 0)) == rowcount + assert model.rowCount(model.index(0, 0)) == rowcount def test_url_completion_delete_bookmark(qtmodeltester, bookmarks, From 9dfc6b0f613edf713163e2b2a33f185390c6d58b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 28 Nov 2017 08:37:38 -0500 Subject: [PATCH 186/415] Change wording for orphaned editor. The term "vanished" is used elsewhere. --- qutebrowser/browser/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 3dcece045..94825882a 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1653,7 +1653,7 @@ class CommandDispatcher: def on_editor_orphaned(self, ed): ed.editing_finished.disconnect() ed.editing_finished.connect( - lambda: message.warning('Edited element was closed')) + lambda: message.warning('Edited element vanished')) @cmdutils.register(instance='command-dispatcher', maxsplit=0, scope='window') From 01a940539126f70424052892a7dfe3db44d86a63 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 28 Nov 2017 08:38:04 -0500 Subject: [PATCH 187/415] Make ophaned editor test less flaky. Instead of having the editor sleep a short time, explicitly send it a signal to exit. --- tests/end2end/features/editor.feature | 5 ++-- tests/end2end/features/test_editor_bdd.py | 28 +++++++++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 92de6ccdd..13e3deddc 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -116,14 +116,15 @@ Feature: Opening external editors Then the javascript message "text: foobar" should be logged Scenario: Spawning an editor and closing the tab - When I set up a fake editor returning "foo" after 1s + When I set up a fake editor that waits And I open data/editor.html And I run :click-element id qute-textarea And I wait for "Entering mode KeyMode.insert (reason: clicking input)" in the log And I run :open-editor And I set tabs.last_close to blank And I run :tab-close - Then the warning "Edited element was closed" should be shown + And I kill the waiting editor + Then the warning "Edited element vanished" should be shown @qtwebengine_todo: Caret mode is not implemented yet Scenario: Spawning an editor in caret mode diff --git a/tests/end2end/features/test_editor_bdd.py b/tests/end2end/features/test_editor_bdd.py index 0cd85a351..3421be88c 100644 --- a/tests/end2end/features/test_editor_bdd.py +++ b/tests/end2end/features/test_editor_bdd.py @@ -20,6 +20,8 @@ import sys import json import textwrap +import os +import signal import pytest_bdd as bdd bdd.scenarios('editor.feature') @@ -66,19 +68,31 @@ def set_up_editor_empty(quteproc, server, tmpdir): set_up_editor(quteproc, server, tmpdir, "") -@bdd.when(bdd.parsers.parse( - 'I set up a fake editor returning "{text}" after {t}s')) -def set_up_editor_delay(quteproc, server, tmpdir, text, t): +@bdd.when(bdd.parsers.parse('I set up a fake editor that waits')) +def set_up_editor_wait(quteproc, server, tmpdir): """Set up editor.command to a small python script inserting a text.""" + pidfile = tmpdir / 'editor_pid' script = tmpdir / 'script.py' script.write(textwrap.dedent(""" + import os import sys import time + import signal - time.sleep({t}) + with open('{pidfile}', 'w') as f: + f.write(str(os.getpid())) - with open(sys.argv[1], 'w', encoding='utf-8') as f: - f.write({text!r}) - """.format(text=text, t=t))) + signal.signal(signal.SIGUSR1, lambda s, f: sys.exit(0)) + + time.sleep(100) + """.format(pidfile=pidfile))) editor = json.dumps([sys.executable, str(script), '{}']) quteproc.set_setting('editor.command', editor) + + +@bdd.when(bdd.parsers.parse('I kill the waiting editor')) +def kill_editor_wait(quteproc, server, tmpdir): + """Kill the waiting editor.""" + pidfile = tmpdir / 'editor_pid' + pid = int(pidfile.read()) + os.kill(pid, signal.SIGUSR1) From ca74991900310926f343b79c3845eabab9dea9b1 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 16:58:18 +0100 Subject: [PATCH 188/415] Read backend after args.temp_settings This should hopefully not affect any config change handlers, as almost nothing is registered this early. Fixes #3340 --- doc/changelog.asciidoc | 1 + qutebrowser/config/configinit.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 722001ef8..087619870 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -104,6 +104,7 @@ Fixed - Fixed crash when opening `qute://help/img` - Fixed `gU` (`:navigate up`) on `qute://help` and webservers not handling `..` in a URL. +- Using e.g. `-s backend webkit` to set the backend now works correctly. Deprecated ~~~~~~~~~~ diff --git a/qutebrowser/config/configinit.py b/qutebrowser/config/configinit.py index 6aaf40720..510245e2e 100644 --- a/qutebrowser/config/configinit.py +++ b/qutebrowser/config/configinit.py @@ -66,14 +66,14 @@ def early_init(args): configfiles.init() - objects.backend = get_backend(args) - for opt, val in args.temp_settings: try: config.instance.set_str(opt, val) except configexc.Error as e: message.error("set: {} - {}".format(e.__class__.__name__, e)) + objects.backend = get_backend(args) + configtypes.Font.monospace_fonts = config.val.fonts.monospace config.instance.changed.connect(_update_monospace_fonts) From 3857491cf9b7a2c686c5a70a68e77b8f25aa89b4 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 17:06:41 +0100 Subject: [PATCH 189/415] Make sure docs are up-to-date for navigate.feature --- tests/end2end/features/navigate.feature | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/end2end/features/navigate.feature b/tests/end2end/features/navigate.feature index 307a6c53a..ff6bc121b 100644 --- a/tests/end2end/features/navigate.feature +++ b/tests/end2end/features/navigate.feature @@ -19,7 +19,8 @@ Feature: Using :navigate Then data/navigate should be loaded Scenario: Navigating up in qute://help/ - When I open qute://help/commands.html + When the documentation is up to date + And I open qute://help/commands.html And I run :navigate up Then qute://help/ should be loaded From 26edf55f852367c95835ee27ba89b5ecbd92e462 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 17:56:17 +0100 Subject: [PATCH 190/415] Update install instructions for Fedora/OpenBSD --- doc/install.asciidoc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/install.asciidoc b/doc/install.asciidoc index cdbfa5d40..1dba5fa57 100644 --- a/doc/install.asciidoc +++ b/doc/install.asciidoc @@ -78,10 +78,17 @@ $ python3 scripts/asciidoc2html.py On Fedora --------- -The Fedora packages are lagging behind a lot and are currently effectively -unmaintained. It's recommended to <> instead. +NOTE: Fedora's packages used to be outdated for a long time, but are +now (November 2017) maintained and up-to-date again. -Related Fedora bug: https://bugzilla.redhat.com/show_bug.cgi?id=1467748[1467748] +qutebrowser is available in the official repositories: + +----- +# dnf install qutebrowser +----- + +However, note that Fedora 25/26 won't be updated to qutebrowser v1.0, so you +might want to <> instead there. On Archlinux ------------ @@ -182,6 +189,10 @@ To use the QtWebEngine backend, install `libqt5-qtwebengine`. On OpenBSD ---------- +WARNING: OpenBSD only packages a legacy unmaintained version of QtWebKit (for +which support was dropped in qutebrowser v1.0). It's advised to not use +qutebrowser from OpenBSD ports for untrusted websites. + qutebrowser is in http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/www/qutebrowser/[OpenBSD ports]. Install the package: From 43ff35eaea5b4070d20570652211c8f79705a023 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 28 Nov 2017 22:15:56 +0100 Subject: [PATCH 191/415] Revert "Make sure docs are up-to-date for navigate.feature" This action doesn't exist in navigate.feature This reverts commit 3857491cf9b7a2c686c5a70a68e77b8f25aa89b4. --- tests/end2end/features/navigate.feature | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/end2end/features/navigate.feature b/tests/end2end/features/navigate.feature index ff6bc121b..307a6c53a 100644 --- a/tests/end2end/features/navigate.feature +++ b/tests/end2end/features/navigate.feature @@ -19,8 +19,7 @@ Feature: Using :navigate Then data/navigate should be loaded Scenario: Navigating up in qute://help/ - When the documentation is up to date - And I open qute://help/commands.html + When I open qute://help/commands.html And I run :navigate up Then qute://help/ should be loaded From 19cecbdeaecbe740bab02adeaebd88c2cf4998d9 Mon Sep 17 00:00:00 2001 From: Jay Kamat Date: Tue, 28 Nov 2017 22:08:22 -0500 Subject: [PATCH 192/415] Fix style issues in scattered hint tests --- tests/end2end/test_hints_html.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/end2end/test_hints_html.py b/tests/end2end/test_hints_html.py index 7b84bd635..5e4ad3837 100644 --- a/tests/end2end/test_hints_html.py +++ b/tests/end2end/test_hints_html.py @@ -158,8 +158,14 @@ def test_word_hints_issue1393(quteproc, tmpdir): @pytest.mark.parametrize('num_elements', range(1, 26)) def test_scattered_hints_count(win_registry, mode_manager, min_len, num_chars, num_elements): - """Test scattered hints function.""" - # pylint: disable=W0141 + """Test scattered hints function. + + Tests many properties from an invocation of _hint_scattered, including + + 1. Hints must be unique + 2. There can only be two hint lengths, only 1 apart + 3. There are no unique prefixes for long hints, such as 'la' with no 'l' + """ manager = qutebrowser.browser.hints.HintManager(0, 0) chars = string.ascii_lowercase[:num_chars] @@ -170,11 +176,11 @@ def test_scattered_hints_count(win_registry, mode_manager, min_len, assert len(hints) == len(set(hints)) # Check if any hints are shorter than min_len - assert not any([x for x in hints if len(x) < min_len]) + assert not any(x for x in hints if len(x) < min_len) # Check we don't have more than 2 link lengths # Eg: 'a' 'bc' and 'def' cannot be in the same hint string - hint_lens = set(map(len, hints)) + hint_lens = {len(h) for h in hints} assert len(hint_lens) <= 2 if len(hint_lens) == 2: @@ -183,7 +189,7 @@ def test_scattered_hints_count(win_registry, mode_manager, min_len, # 'ab' and 'c' can assert abs(functools.reduce(operator.sub, hint_lens)) <= 1 - longest_hints = filter(lambda x: len(x) == max(hint_lens), hints) + longest_hints = [x for x in hints if len(x) == max(hint_lens)] if min_len < max(hint_lens) - 1: # Check if we have any unique prefixes. For example, 'la' @@ -192,4 +198,4 @@ def test_scattered_hints_count(win_registry, mode_manager, min_len, for x in longest_hints: prefix = x[:-1] count_map[prefix] = count_map.get(prefix, 0) + 1 - assert not any(filter(lambda x: x == 1, count_map.values())) + assert all(e != 1 for e in count_map.values()) From d4291dd4aefa1d37111832c27839e1b4e73c1e95 Mon Sep 17 00:00:00 2001 From: Rouji Date: Wed, 29 Nov 2017 08:40:40 +0100 Subject: [PATCH 193/415] don't 'exec' printf --- scripts/open_url_in_instance.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/open_url_in_instance.sh b/scripts/open_url_in_instance.sh index 85cd919d4..c4acf57a1 100755 --- a/scripts/open_url_in_instance.sh +++ b/scripts/open_url_in_instance.sh @@ -8,8 +8,8 @@ _proto_version=1 _ipc_socket="${XDG_RUNTIME_DIR}/qutebrowser/ipc-$(echo -n "$USER" | md5sum | cut -d' ' -f1)" _qute_bin="/usr/bin/qutebrowser" -exec printf '{"args": ["%s"], "target_arg": null, "version": "%s", "protocol_version": %d, "cwd": "%s"}\n' \ - "${_url}" \ - "${_qb_version}" \ - "${_proto_version}" \ - "${PWD}" | socat - UNIX-CONNECT:"${_ipc_socket}" 2>/dev/null || $_qute_bin --backend webengine "$@" & +printf '{"args": ["%s"], "target_arg": null, "version": "%s", "protocol_version": %d, "cwd": "%s"}\n' \ + "${_url}" \ + "${_qb_version}" \ + "${_proto_version}" \ + "${PWD}" | socat - UNIX-CONNECT:"${_ipc_socket}" 2>/dev/null || $_qute_bin --backend webengine "$@" & From 938198e92aea1a140523d326df2e253b50d7a3fc Mon Sep 17 00:00:00 2001 From: Rouji Date: Wed, 29 Nov 2017 08:43:02 +0100 Subject: [PATCH 194/415] quote _qute_bin (paths may contain whitespace), remove unnecessary --backend argument --- scripts/open_url_in_instance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/open_url_in_instance.sh b/scripts/open_url_in_instance.sh index c4acf57a1..7e07c1394 100755 --- a/scripts/open_url_in_instance.sh +++ b/scripts/open_url_in_instance.sh @@ -12,4 +12,4 @@ printf '{"args": ["%s"], "target_arg": null, "version": "%s", "protocol_version" "${_url}" \ "${_qb_version}" \ "${_proto_version}" \ - "${PWD}" | socat - UNIX-CONNECT:"${_ipc_socket}" 2>/dev/null || $_qute_bin --backend webengine "$@" & + "${PWD}" | socat - UNIX-CONNECT:"${_ipc_socket}" 2>/dev/null || "$_qute_bin" "$@" & From afb4a6be51b3126563c764b8598e9d46e9b9d1bb Mon Sep 17 00:00:00 2001 From: Rouji Date: Wed, 29 Nov 2017 08:43:46 +0100 Subject: [PATCH 195/415] bump _qb_version --- scripts/open_url_in_instance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/open_url_in_instance.sh b/scripts/open_url_in_instance.sh index 7e07c1394..6abd8b424 100755 --- a/scripts/open_url_in_instance.sh +++ b/scripts/open_url_in_instance.sh @@ -3,7 +3,7 @@ # author: Thore Bödecker (foxxx0) _url="$1" -_qb_version='0.10.1' +_qb_version='1.0.3' _proto_version=1 _ipc_socket="${XDG_RUNTIME_DIR}/qutebrowser/ipc-$(echo -n "$USER" | md5sum | cut -d' ' -f1)" _qute_bin="/usr/bin/qutebrowser" From edba3f83bc566e2f52a95921876f7beb5555af12 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 29 Nov 2017 10:38:09 +0100 Subject: [PATCH 196/415] Bump open_url_in_instance version to 1.0.4 --- scripts/open_url_in_instance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/open_url_in_instance.sh b/scripts/open_url_in_instance.sh index 6abd8b424..a6ce0ed91 100755 --- a/scripts/open_url_in_instance.sh +++ b/scripts/open_url_in_instance.sh @@ -3,7 +3,7 @@ # author: Thore Bödecker (foxxx0) _url="$1" -_qb_version='1.0.3' +_qb_version='1.0.4' _proto_version=1 _ipc_socket="${XDG_RUNTIME_DIR}/qutebrowser/ipc-$(echo -n "$USER" | md5sum | cut -d' ' -f1)" _qute_bin="/usr/bin/qutebrowser" From af638ec430be25c5f61c697a49efaeb7cade2bd6 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 29 Nov 2017 10:44:32 +0100 Subject: [PATCH 197/415] Move hint unittests to their own file --- tests/end2end/test_hints_html.py | 53 ----------------------- tests/unit/browser/test_hints.py | 74 ++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 53 deletions(-) create mode 100644 tests/unit/browser/test_hints.py diff --git a/tests/end2end/test_hints_html.py b/tests/end2end/test_hints_html.py index 5e4ad3837..abc106505 100644 --- a/tests/end2end/test_hints_html.py +++ b/tests/end2end/test_hints_html.py @@ -22,17 +22,12 @@ import os import os.path import textwrap -import string -import functools -import operator import attr import yaml import pytest import bs4 -import qutebrowser.browser.hints - def collect_tests(): basedir = os.path.dirname(__file__) @@ -151,51 +146,3 @@ def test_word_hints_issue1393(quteproc, tmpdir): quteproc.wait_for(message='hints: *', category='hints') quteproc.send_cmd(':follow-hint {}'.format(hint)) quteproc.wait_for_load_finished('data/{}'.format(target)) - - -@pytest.mark.parametrize('min_len', [0, 3]) -@pytest.mark.parametrize('num_chars', [9]) -@pytest.mark.parametrize('num_elements', range(1, 26)) -def test_scattered_hints_count(win_registry, mode_manager, min_len, - num_chars, num_elements): - """Test scattered hints function. - - Tests many properties from an invocation of _hint_scattered, including - - 1. Hints must be unique - 2. There can only be two hint lengths, only 1 apart - 3. There are no unique prefixes for long hints, such as 'la' with no 'l' - """ - manager = qutebrowser.browser.hints.HintManager(0, 0) - chars = string.ascii_lowercase[:num_chars] - - hints = manager._hint_scattered(min_len, chars, - list(range(num_elements))) - - # Check if hints are unique - assert len(hints) == len(set(hints)) - - # Check if any hints are shorter than min_len - assert not any(x for x in hints if len(x) < min_len) - - # Check we don't have more than 2 link lengths - # Eg: 'a' 'bc' and 'def' cannot be in the same hint string - hint_lens = {len(h) for h in hints} - assert len(hint_lens) <= 2 - - if len(hint_lens) == 2: - # Check if hint_lens are more than 1 apart - # Eg: 'abc' and 'd' cannot be in the same hint sequence, but - # 'ab' and 'c' can - assert abs(functools.reduce(operator.sub, hint_lens)) <= 1 - - longest_hints = [x for x in hints if len(x) == max(hint_lens)] - - if min_len < max(hint_lens) - 1: - # Check if we have any unique prefixes. For example, 'la' - # alone, with no other 'l' - count_map = {} - for x in longest_hints: - prefix = x[:-1] - count_map[prefix] = count_map.get(prefix, 0) + 1 - assert all(e != 1 for e in count_map.values()) diff --git a/tests/unit/browser/test_hints.py b/tests/unit/browser/test_hints.py new file mode 100644 index 000000000..6792d1754 --- /dev/null +++ b/tests/unit/browser/test_hints.py @@ -0,0 +1,74 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +import string +import functools +import operator + +import pytest + +import qutebrowser.browser.hints + + +@pytest.mark.parametrize('min_len', [0, 3]) +@pytest.mark.parametrize('num_chars', [9]) +@pytest.mark.parametrize('num_elements', range(1, 26)) +def test_scattered_hints_count(win_registry, mode_manager, min_len, + num_chars, num_elements): + """Test scattered hints function. + + Tests many properties from an invocation of _hint_scattered, including + + 1. Hints must be unique + 2. There can only be two hint lengths, only 1 apart + 3. There are no unique prefixes for long hints, such as 'la' with no 'l' + """ + manager = qutebrowser.browser.hints.HintManager(0, 0) + chars = string.ascii_lowercase[:num_chars] + + hints = manager._hint_scattered(min_len, chars, + list(range(num_elements))) + + # Check if hints are unique + assert len(hints) == len(set(hints)) + + # Check if any hints are shorter than min_len + assert not any(x for x in hints if len(x) < min_len) + + # Check we don't have more than 2 link lengths + # Eg: 'a' 'bc' and 'def' cannot be in the same hint string + hint_lens = {len(h) for h in hints} + assert len(hint_lens) <= 2 + + if len(hint_lens) == 2: + # Check if hint_lens are more than 1 apart + # Eg: 'abc' and 'd' cannot be in the same hint sequence, but + # 'ab' and 'c' can + assert abs(functools.reduce(operator.sub, hint_lens)) <= 1 + + longest_hints = [x for x in hints if len(x) == max(hint_lens)] + + if min_len < max(hint_lens) - 1: + # Check if we have any unique prefixes. For example, 'la' + # alone, with no other 'l' + count_map = {} + for x in longest_hints: + prefix = x[:-1] + count_map[prefix] = count_map.get(prefix, 0) + 1 + assert all(e != 1 for e in count_map.values()) From 81045fb1bd88a2dc9c6136ee130d54929d751881 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 29 Nov 2017 10:44:50 +0100 Subject: [PATCH 198/415] Drop unneeded parens --- qutebrowser/browser/hints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/hints.py b/qutebrowser/browser/hints.py index b68d00058..6d40f531a 100644 --- a/qutebrowser/browser/hints.py +++ b/qutebrowser/browser/hints.py @@ -451,7 +451,7 @@ class HintManager(QObject): # dividing by the number of spots we would loose by adding a # short element short_count = math.floor((total_space - len(elems)) / - (len(chars))) + len(chars)) # Check if we double counted above to warrant another short_count # https://github.com/qutebrowser/qutebrowser/issues/3242 if total_space - (short_count * len(chars) + From e09a8c77e9688c4610367ab689aa4067004e8bbe Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 29 Nov 2017 07:01:48 -0500 Subject: [PATCH 199/415] Simplify check for orphaned editor. Instead of rewiring signals on tab.shutting_down, have the webelem check if its parent tab is deleted, and throw a specific exception. This is only necessary in WebEngine, Webkit does not crash when the editor is orphaned. I tried to write a test for is_deleted, but could not get it to pass: ``` def test_is_deleted(qtbot, view, config_stub, tab_registry, mode_manager): tab_w = Tab(win_id=0, mode_manager=mode_manager) qtbot.add_widget(tab_w) tab_w._set_widget(view) assert not tab_w.is_deleted() sip.delete(view) #assert tab_w.is_deleted() ``` The qtbot post-test cleanup would error due to the deleted view. --- qutebrowser/browser/browsertab.py | 4 ++++ qutebrowser/browser/commands.py | 10 ++-------- qutebrowser/browser/webelem.py | 7 +++++++ qutebrowser/browser/webengine/webengineelem.py | 2 ++ 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index 547e276db..c8e595baf 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -22,6 +22,7 @@ import enum import itertools +import sip import attr from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QSizeF, Qt from PyQt5.QtGui import QIcon @@ -864,3 +865,6 @@ class AbstractTab(QWidget): except (AttributeError, RuntimeError) as exc: url = '<{}>'.format(exc.__class__.__name__) return utils.get_repr(self, tab_id=self.tab_id, url=url) + + def is_deleted(self): + return sip.isdeleted(self._widget) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 94825882a..781dfeb24 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1621,9 +1621,6 @@ class CommandDispatcher: ed = editor.ExternalEditor(self._tabbed_browser) ed.editing_finished.connect(functools.partial( self.on_editing_finished, elem)) - tab = self._current_widget() - tab.shutting_down.connect(functools.partial( - self.on_editor_orphaned, ed)) ed.edit(text, caret_position) @cmdutils.register(instance='command-dispatcher', scope='window') @@ -1647,14 +1644,11 @@ class CommandDispatcher: """ try: elem.set_value(text) + except webelem.OrphanedError as e: + message.warning('Edited element vanished') except webelem.Error as e: raise cmdexc.CommandError(str(e)) - def on_editor_orphaned(self, ed): - ed.editing_finished.disconnect() - ed.editing_finished.connect( - lambda: message.warning('Edited element vanished')) - @cmdutils.register(instance='command-dispatcher', maxsplit=0, scope='window') def insert_text(self, text): diff --git a/qutebrowser/browser/webelem.py b/qutebrowser/browser/webelem.py index 067f33cff..10496cac7 100644 --- a/qutebrowser/browser/webelem.py +++ b/qutebrowser/browser/webelem.py @@ -60,6 +60,13 @@ class Error(Exception): pass +class OrphanedError(Exception): + + """Raised when a webelement's parent has vanished.""" + + pass + + class AbstractWebElement(collections.abc.MutableMapping): """A wrapper around QtWebKit/QtWebEngine web element. diff --git a/qutebrowser/browser/webengine/webengineelem.py b/qutebrowser/browser/webengine/webengineelem.py index bdb8b4192..0e54ca80e 100644 --- a/qutebrowser/browser/webengine/webengineelem.py +++ b/qutebrowser/browser/webengine/webengineelem.py @@ -100,6 +100,8 @@ class WebEngineElement(webelem.AbstractWebElement): def _js_call(self, name, *args, callback=None): """Wrapper to run stuff from webelem.js.""" + if self._tab.is_deleted(): + raise webelem.OrphanedError("Tab containing element vanished") js_code = javascript.assemble('webelem', name, self._id, *args) self._tab.run_js_async(js_code, callback=callback) From fcad40ceb709988ac3a8c4a30e32c95d7ba87a45 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 29 Nov 2017 07:35:11 -0500 Subject: [PATCH 200/415] Add orphaned tab check to WebKit as well. This implements the orphaned editor fix for WebKit. Webkit wasn't crashing before, but this causes webkit to show the same warning webengine does if the editor is orhpaned (rather than silently continuing). This allows the same BDD test to pass for both webkit and webengine. --- qutebrowser/browser/webkit/webkitelem.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qutebrowser/browser/webkit/webkitelem.py b/qutebrowser/browser/webkit/webkitelem.py index 2a1eafc9e..829052798 100644 --- a/qutebrowser/browser/webkit/webkitelem.py +++ b/qutebrowser/browser/webkit/webkitelem.py @@ -118,6 +118,8 @@ class WebKitElement(webelem.AbstractWebElement): def set_value(self, value): self._check_vanished() + if self._tab.is_deleted(): + raise webelem.OrphanedError("Tab containing element vanished") if self.is_content_editable(): log.webelem.debug("Filling {!r} via set_text.".format(self)) self._elem.setPlainText(value) From b465c109eeeae38189e52c5498d6a884b11658e8 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Wed, 29 Nov 2017 07:41:02 -0500 Subject: [PATCH 201/415] Fix test_editor_bdd.py for windows. - Use a raw string to handle windows path separators - Use SIGTERM instead of SIGUSR1 --- tests/end2end/features/test_editor_bdd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/end2end/features/test_editor_bdd.py b/tests/end2end/features/test_editor_bdd.py index 3421be88c..208ba60c4 100644 --- a/tests/end2end/features/test_editor_bdd.py +++ b/tests/end2end/features/test_editor_bdd.py @@ -79,10 +79,10 @@ def set_up_editor_wait(quteproc, server, tmpdir): import time import signal - with open('{pidfile}', 'w') as f: + with open(r'{pidfile}', 'w') as f: f.write(str(os.getpid())) - signal.signal(signal.SIGUSR1, lambda s, f: sys.exit(0)) + signal.signal(signal.SIGTERM, lambda s, f: sys.exit(0)) time.sleep(100) """.format(pidfile=pidfile))) @@ -95,4 +95,4 @@ def kill_editor_wait(quteproc, server, tmpdir): """Kill the waiting editor.""" pidfile = tmpdir / 'editor_pid' pid = int(pidfile.read()) - os.kill(pid, signal.SIGUSR1) + os.kill(pid, signal.SIGTERM) From 2752055281e4117d42288cc39abae7121d538a48 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 29 Nov 2017 22:17:48 +0100 Subject: [PATCH 202/415] Fix lint in webserver_sub.py --- tests/end2end/fixtures/webserver_sub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/end2end/fixtures/webserver_sub.py b/tests/end2end/fixtures/webserver_sub.py index 4ec929203..6e0b740ba 100644 --- a/tests/end2end/fixtures/webserver_sub.py +++ b/tests/end2end/fixtures/webserver_sub.py @@ -62,7 +62,7 @@ def send_data(path): data_dir = os.path.join(basedir, 'data') print(basedir) if os.path.isdir(os.path.join(data_dir, path)): - path = path + '/index.html' + path += '/index.html' return flask.send_from_directory(data_dir, path) @@ -215,7 +215,7 @@ def drip(): def generate_bytes(): for _ in range(numbytes): - yield u"*".encode('utf-8') + yield "*".encode('utf-8') time.sleep(pause) response = flask.Response(generate_bytes(), headers={ From ac89ab23a915a3cc6b0d793121c7e07270c890a9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 29 Nov 2017 22:36:06 +0100 Subject: [PATCH 203/415] Revert "Add nEXT to list of alternatives" It uses a buggy legacy WebKit... This reverts commit 814a88b741eac871b64eb8acb712651b53d06958. --- README.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/README.asciidoc b/README.asciidoc index 8ba51ca98..a87165366 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -174,7 +174,6 @@ Active * https://luakit.github.io/luakit/[luakit] (C/Lua, GTK+ with WebKit2) * http://surf.suckless.org/[surf] (C, GTK+ with WebKit1/WebKit2) * http://www.uzbl.org/[uzbl] (C, GTK+ with WebKit1/WebKit2) -* https://github.com/nEXT-Browser/nEXT[nEXT] (Lisp, emacs-like, currently macOS only) * Chrome/Chromium addons: https://github.com/1995eaton/chromium-vim[cVim], http://vimium.github.io/[Vimium], From d29cf1ee4d0cf68b417d77de617ae7fbe24e3814 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Thu, 30 Nov 2017 00:09:28 +0100 Subject: [PATCH 204/415] lazy sessions, restore if visible, forward user after restore --- qutebrowser/html/back.html | 53 ++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index 83f5cb74e..94d16824e 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -1,16 +1,53 @@ {% extends "base.html" %} {% block script %} -setTimeout(function() { - /* drop first focus event, to avoid problems - (allow to go easily to newer history entries) */ - window.onfocus = function() { - window.onfocus = null; - window.history.back(); - }; -}, 1000); +const STATE_BACK = "back"; +const STATE_FORWARD = "forward"; + +function switch_state(new_state) { + history.replaceState( + new_state, + document.title, + location.pathname+location.hash); +} + +function go_back() { + switch_state(STATE_FORWARD); + history.back(); +} +function go_forward() { + switch_state(STATE_BACK); + history.forward(); +} + +// there are three states +// default: register focus listener, +// on focus: go back and switch to the state forward +// back: user came from a later history entry +// -> switch to the state forward, +// forward him to the previous history entry +// forward: user came from a previous history entry +// -> switch to the state back, +// forward him to the next history entry +switch (history.state) { + case STATE_BACK: + go_back(); + break; + case STATE_FORWARD: + go_forward(); + break; + default: + if (!document.hidden) { + go_back(); + break; + } + + document.addEventListener("visibilitychange", go_back); + break; +} {% endblock %} {% block content %} +

If you see this site something went wrong or you reached the end of the history or you disabled javascript.

{% endblock %} From 45a1989a1f544634cf4ad3a70fc0aa62cb4d52bf Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 30 Nov 2017 01:26:44 +0100 Subject: [PATCH 205/415] Add some more links to initial readme line [ci skip] --- README.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.asciidoc b/README.asciidoc index a87165366..a625f317c 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -15,7 +15,7 @@ image:https://travis-ci.org/qutebrowser/qutebrowser.svg?branch=master["Build Sta image:https://ci.appveyor.com/api/projects/status/5pyauww2k68bbow2/branch/master?svg=true["AppVeyor build status", link="https://ci.appveyor.com/project/qutebrowser/qutebrowser"] image:https://codecov.io/github/qutebrowser/qutebrowser/coverage.svg?branch=master["coverage badge",link="https://codecov.io/github/qutebrowser/qutebrowser?branch=master"] -link:https://www.qutebrowser.org[website] | link:https://blog.qutebrowser.org[blog] | link:https://github.com/qutebrowser/qutebrowser/releases[releases] +link:https://www.qutebrowser.org[website] | link:https://blog.qutebrowser.org[blog] | https://github.com/qutebrowser/qutebrowser/blob/master/doc/faq.asciidoc[FAQ] | https://www.qutebrowser.org/doc/contributing.html[contributing] | link:https://github.com/qutebrowser/qutebrowser/releases[releases] | https://github.com/qutebrowser/qutebrowser/blob/master/doc/install.asciidoc[installing] // QUTE_WEB_HIDE_END qutebrowser is a keyboard-focused browser with a minimal GUI. It's based From 4497f710f96cc2e2251b3b395d76da7b17ebc698 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Thu, 30 Nov 2017 07:19:27 -0500 Subject: [PATCH 206/415] Try SIGINT instead of SIGTERM for windows. --- tests/end2end/features/test_editor_bdd.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/end2end/features/test_editor_bdd.py b/tests/end2end/features/test_editor_bdd.py index 208ba60c4..34c7e4248 100644 --- a/tests/end2end/features/test_editor_bdd.py +++ b/tests/end2end/features/test_editor_bdd.py @@ -82,9 +82,12 @@ def set_up_editor_wait(quteproc, server, tmpdir): with open(r'{pidfile}', 'w') as f: f.write(str(os.getpid())) - signal.signal(signal.SIGTERM, lambda s, f: sys.exit(0)) + signal.signal(signal.SIGINT, lambda s, f: sys.exit(0)) - time.sleep(100) + try: + time.sleep(100) + except InterruptedError: + pass """.format(pidfile=pidfile))) editor = json.dumps([sys.executable, str(script), '{}']) quteproc.set_setting('editor.command', editor) @@ -95,4 +98,4 @@ def kill_editor_wait(quteproc, server, tmpdir): """Kill the waiting editor.""" pidfile = tmpdir / 'editor_pid' pid = int(pidfile.read()) - os.kill(pid, signal.SIGTERM) + os.kill(pid, signal.SIGINT) From f7b0ac503ec7339109b04abb5e03b4462a10f365 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Wed, 29 Nov 2017 02:29:53 -0600 Subject: [PATCH 207/415] generate pytest envs with tox factors This eliminates all separate pytest envs in favor of conditionals in [testenv]. This requires renaming some environments to make the lack of certain functionality explicit: - instead of omitting pyqt{version}, use pyqtlink to use host PyQt tox.ini: eliminate -nocov It is possible to set the `PYTEST_ADDOPTS` environment variable to enable coverage checks, rather than a new command. --- tox.ini | 142 ++++++++++++++------------------------------------------ 1 file changed, 34 insertions(+), 108 deletions(-) diff --git a/tox.ini b/tox.ini index c0395a621..f805e006b 100644 --- a/tox.ini +++ b/tox.ini @@ -13,120 +13,24 @@ skipsdist = true setenv = QT_QPA_PLATFORM_PLUGIN_PATH={envdir}/Lib/site-packages/PyQt5/plugins/platforms PYTEST_QT_API=pyqt5 + pyqt{,56,571,58,59}: QUTE_BDD_WEBENGINE=true + cov: PYTEST_ADDOPTS=--cov --cov-report xml --cov-report=html --cov-report= passenv = PYTHON DISPLAY XAUTHORITY HOME USERNAME USER CI TRAVIS XDG_* QUTE_* DOCKER +basepython = + py35: python3.5 + py36: python3.6 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-tests.txt + pyqt: PyQt5 + pyqt56: PyQt5==5.6 + pyqt571: PyQt5==5.7.1 + pyqt58: PyQt5==5.8.2 + pyqt59: PyQt5==5.9.2 commands = - {envpython} scripts/link_pyqt.py --tox {envdir} + pyqtlink: {envpython} scripts/link_pyqt.py --tox {envdir} {envpython} -bb -m pytest {posargs:tests} - -# test envs with PyQt5 from PyPI - -[testenv:py35-pyqt56] -basepython = python3.5 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.6 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py35-pyqt571] -basepython = python3.5 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.7.1 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py36-pyqt571] -basepython = python3.6 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.7.1 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py35-pyqt58] -basepython = python3.5 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.8.2 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py36-pyqt58] -basepython = {env:PYTHON:python3.6} -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.8.2 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py35-pyqt59] -basepython = python3.5 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.9.2 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py36-pyqt59] -basepython = {env:PYTHON:python3.6} -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.9.2 -commands = {envpython} -bb -m pytest {posargs:tests} - -# test envs with coverage - -[testenv:py35-pyqt59-cov] -basepython = python3.5 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.9.2 -commands = - {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests} - {envpython} scripts/dev/check_coverage.py {posargs} - -[testenv:py36-pyqt59-cov] -basepython = python3.6 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.9.2 -commands = - {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests} - {envpython} scripts/dev/check_coverage.py {posargs} + cov: {envpython} scripts/dev/check_coverage.py {posargs} # other envs @@ -196,6 +100,16 @@ setenv = PYTHONPATH={toxinidir} commands = {envpython} scripts/dev/run_vulture.py +[testenv:vulture-pyqtlink] +basepython = python3 +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/misc/requirements/requirements-vulture.txt +setenv = PYTHONPATH={toxinidir} +commands = + {envpython} scripts/link_pyqt.py --tox {envdir} + {envpython} scripts/dev/run_vulture.py + [testenv:pylint] basepython = {env:PYTHON:python3} ignore_errors = true @@ -208,6 +122,18 @@ commands = {envpython} -m pylint scripts qutebrowser --output-format=colorized --reports=no {posargs} {envpython} scripts/dev/run_pylint_on_tests.py {toxinidir} --output-format=colorized --reports=no {posargs} +[testenv:pylint-pyqtlink] +basepython = {env:PYTHON:python3} +ignore_errors = true +passenv = +deps = + {[testenv]deps} + -r{toxinidir}/misc/requirements/requirements-pylint.txt +commands = + {envpython} scripts/link_pyqt.py --tox {envdir} + {envpython} -m pylint scripts qutebrowser --output-format=colorized --reports=no {posargs} + {envpython} scripts/dev/run_pylint_on_tests.py {toxinidir} --output-format=colorized --reports=no {posargs} + [testenv:pylint-master] basepython = python3 passenv = {[testenv:pylint]passenv} From 2f231c86ac76cd47038759331b3a1b26418043e0 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Thu, 30 Nov 2017 08:35:02 -0600 Subject: [PATCH 208/415] update tox env name in CI config Use py36-pyqtlink instead of py36 for macOS --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 65d917d73..e92df0b6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ matrix: - os: linux env: TESTENV=py36-pyqt59-cov - os: osx - env: TESTENV=py36 OSX=sierra + env: TESTENV=py36-pyqtlink OSX=sierra osx_image: xcode8.3 language: generic # https://github.com/qutebrowser/qutebrowser/issues/2013 From b58cfead05b201c3664fde8657556c5b260a9e6a Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Thu, 30 Nov 2017 16:05:01 +0100 Subject: [PATCH 209/415] style fixed --- qutebrowser/browser/qutescheme.py | 4 ++-- qutebrowser/html/back.html | 5 +++-- qutebrowser/misc/sessions.py | 10 +++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 9771f6db1..247e12a78 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -433,8 +433,8 @@ def qute_back(url): Simple page to free ram / lazy load a site, goes back on focusing the tab. """ html = jinja.render( - 'back.html', - title='Suspended: ' + urllib.parse.unquote(url.fragment())) + 'back.html', + title='Suspended: ' + urllib.parse.unquote(url.fragment())) return 'text/html', html diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index 94d16824e..b6190feda 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -8,13 +8,14 @@ function switch_state(new_state) { history.replaceState( new_state, document.title, - location.pathname+location.hash); + location.pathname + location.hash); } function go_back() { switch_state(STATE_FORWARD); history.back(); } + function go_forward() { switch_state(STATE_BACK); history.forward(); @@ -49,5 +50,5 @@ switch (history.state) { {% block content %} -

If you see this site something went wrong or you reached the end of the history or you disabled javascript.

+

If you see this site something went wrong, or you reached the end of the history, or you disabled javascript.

{% endblock %} diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 1b0106f1b..237e46189 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -336,11 +336,11 @@ class SessionManager(QObject): # -> dropwhile empty if not session.lazy_session lazy_index = len(data['history']) gen = itertools.chain( - itertools.takewhile(lambda _: not lazy_load, - enumerate(data['history'])), - enumerate(lazy_load), - itertools.dropwhile(lambda i: i[0] < lazy_index, - enumerate(data['history']))) + itertools.takewhile(lambda _: not lazy_load, + enumerate(data['history'])), + enumerate(lazy_load), + itertools.dropwhile(lambda i: i[0] < lazy_index, + enumerate(data['history']))) for i, histentry in gen: user_data = {} From a5d0b9851ace4a3a335748925cc971b7f7407def Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Thu, 30 Nov 2017 14:14:11 -0600 Subject: [PATCH 210/415] tox.ini: remove pyqt5.6, use requirements-pyqt.txt --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index f805e006b..1c87d1a6c 100644 --- a/tox.ini +++ b/tox.ini @@ -22,8 +22,7 @@ basepython = deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-tests.txt - pyqt: PyQt5 - pyqt56: PyQt5==5.6 + pyqt: -r{toxinidir}/misc/requirements/requirements-pyqt.txt pyqt571: PyQt5==5.7.1 pyqt58: PyQt5==5.8.2 pyqt59: PyQt5==5.9.2 From 6b762037809b9057774bb88f333466d6d1ba9bd3 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Thu, 30 Nov 2017 14:21:37 -0600 Subject: [PATCH 211/415] update contributing.asciidoc with -pyqtlink envs --- doc/contributing.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing.asciidoc b/doc/contributing.asciidoc index afbb752c5..6f0b1bb72 100644 --- a/doc/contributing.asciidoc +++ b/doc/contributing.asciidoc @@ -97,7 +97,7 @@ unittests and several linters/checkers. Currently, the following tox environments are available: * Tests using https://www.pytest.org[pytest]: - - `py35`, `py36`: Run pytest for python 3.5/3.6 with the system-wide PyQt. + - `py35-pyqtlink`, `py36-pyqtlink`: Run pytest for python 3.5/3.6 with the system-wide PyQt. - `py36-pyqt57`, ..., `py36-pyqt59`: Run pytest with the given PyQt version (`py35-*` also works). - `py36-pyqt59-cov`: Run with coverage support (other Python/PyQt versions work too). * `flake8`: Run various linting checks via https://pypi.python.org/pypi/flake8[flake8]. From 49485ca220543ce9fbcc48da9f3b69feef695822 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Thu, 30 Nov 2017 16:58:14 -0600 Subject: [PATCH 212/415] tox.ini: fix conditional syntax errors `{[testenv]deps}` was passing conditionals in their raw form; this simply lists them manually to avoid this. --- tox.ini | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 1c87d1a6c..de41f2e0f 100644 --- a/tox.ini +++ b/tox.ini @@ -107,14 +107,15 @@ deps = setenv = PYTHONPATH={toxinidir} commands = {envpython} scripts/link_pyqt.py --tox {envdir} - {envpython} scripts/dev/run_vulture.py + {[testenv:vulture]commands} [testenv:pylint] basepython = {env:PYTHON:python3} ignore_errors = true passenv = deps = - {[testenv]deps} + -r{toxinidir}/requirements.txt + -r{toxinidir}/misc/requirements/requirements-tests.txt -r{toxinidir}/misc/requirements/requirements-pylint.txt -r{toxinidir}/misc/requirements/requirements-pyqt.txt commands = @@ -126,18 +127,19 @@ basepython = {env:PYTHON:python3} ignore_errors = true passenv = deps = - {[testenv]deps} + -r{toxinidir}/requirements.txt + -r{toxinidir}/misc/requirements/requirements-tests.txt -r{toxinidir}/misc/requirements/requirements-pylint.txt commands = {envpython} scripts/link_pyqt.py --tox {envdir} - {envpython} -m pylint scripts qutebrowser --output-format=colorized --reports=no {posargs} - {envpython} scripts/dev/run_pylint_on_tests.py {toxinidir} --output-format=colorized --reports=no {posargs} + {[testenv:pylint]commands} [testenv:pylint-master] basepython = python3 passenv = {[testenv:pylint]passenv} deps = - {[testenv]deps} + -r{toxinidir}/requirements.txt + -r{toxinidir}/misc/requirements/requirements-tests.txt -r{toxinidir}/misc/requirements/requirements-pylint-master.txt commands = {envpython} scripts/link_pyqt.py --tox {envdir} From 822f6bae2cceff6e5cca4ab10a3d4cfbdf52a5b8 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 1 Dec 2017 07:25:21 -0500 Subject: [PATCH 213/415] Fix webkitelem test. Now that it checks tab.is_deleted we need to mock that. --- tests/unit/browser/webkit/test_webkitelem.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/browser/webkit/test_webkitelem.py b/tests/unit/browser/webkit/test_webkitelem.py index c64756eb5..35b9c354b 100644 --- a/tests/unit/browser/webkit/test_webkitelem.py +++ b/tests/unit/browser/webkit/test_webkitelem.py @@ -29,7 +29,7 @@ import pytest from PyQt5.QtCore import QRect, QPoint, QUrl QWebElement = pytest.importorskip('PyQt5.QtWebKit').QWebElement -from qutebrowser.browser import webelem +from qutebrowser.browser import webelem, browsertab from qutebrowser.browser.webkit import webkitelem from qutebrowser.misc import objects from qutebrowser.utils import usertypes @@ -127,7 +127,9 @@ def get_webelem(geometry=None, frame=None, *, null=False, style=None, return style_dict[name] elem.styleProperty.side_effect = _style_property - wrapped = webkitelem.WebKitElement(elem, tab=None) + tab = mock.Mock(autospec=browsertab.AbstractTab) + tab.is_deleted.return_value = False + wrapped = webkitelem.WebKitElement(elem, tab=tab) return wrapped From 1102ae4d7e08982e3790430156326a8d7fb2005a Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 1 Dec 2017 07:35:13 -0500 Subject: [PATCH 214/415] Skip editor orphaned test on windows. Tried SIGINT/SIGTERM, neither worked. Just skip this test on windows and go back to SIGUSR1 otherwise. --- tests/end2end/features/editor.feature | 2 ++ tests/end2end/features/test_editor_bdd.py | 10 +++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 13e3deddc..4647eb08c 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -115,6 +115,8 @@ Feature: Opening external editors And I run :click-element id qute-button Then the javascript message "text: foobar" should be logged + # Could not get signals working on Windows + @posix Scenario: Spawning an editor and closing the tab When I set up a fake editor that waits And I open data/editor.html diff --git a/tests/end2end/features/test_editor_bdd.py b/tests/end2end/features/test_editor_bdd.py index 34c7e4248..983ded7ba 100644 --- a/tests/end2end/features/test_editor_bdd.py +++ b/tests/end2end/features/test_editor_bdd.py @@ -82,12 +82,8 @@ def set_up_editor_wait(quteproc, server, tmpdir): with open(r'{pidfile}', 'w') as f: f.write(str(os.getpid())) - signal.signal(signal.SIGINT, lambda s, f: sys.exit(0)) - - try: - time.sleep(100) - except InterruptedError: - pass + signal.signal(signal.SIGUSR1, lambda s, f: sys.exit(0)) + time.sleep(100) """.format(pidfile=pidfile))) editor = json.dumps([sys.executable, str(script), '{}']) quteproc.set_setting('editor.command', editor) @@ -98,4 +94,4 @@ def kill_editor_wait(quteproc, server, tmpdir): """Kill the waiting editor.""" pidfile = tmpdir / 'editor_pid' pid = int(pidfile.read()) - os.kill(pid, signal.SIGINT) + os.kill(pid, signal.SIGUSR1) From 3cfa0f7586ffd1028f69236b4ba0a87de1c48369 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 1 Dec 2017 08:41:08 -0500 Subject: [PATCH 215/415] Make pylint happy for test_editor_bdd. windows has no SIGUSR1, but we don't run this on windows anyways for posix, there IS a member so we need to ignore useless-suppression --- tests/end2end/features/test_editor_bdd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/end2end/features/test_editor_bdd.py b/tests/end2end/features/test_editor_bdd.py index 983ded7ba..1aa253ac1 100644 --- a/tests/end2end/features/test_editor_bdd.py +++ b/tests/end2end/features/test_editor_bdd.py @@ -94,4 +94,7 @@ def kill_editor_wait(quteproc, server, tmpdir): """Kill the waiting editor.""" pidfile = tmpdir / 'editor_pid' pid = int(pidfile.read()) + # windows has no SIGUSR1, but we don't run this on windows anyways + # for posix, there IS a member so we need to ignore useless-suppression + # pylint: disable=no-member,useless-suppression os.kill(pid, signal.SIGUSR1) From 780ac3f4c2bb5bd53d63531215f28dd07613d271 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 1 Dec 2017 11:34:47 -0500 Subject: [PATCH 216/415] Remove needles quteproc/server fixture deps. A few step definitions listed these in the parameters although they were unused. --- tests/end2end/features/test_editor_bdd.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/end2end/features/test_editor_bdd.py b/tests/end2end/features/test_editor_bdd.py index 983ded7ba..ec43c8477 100644 --- a/tests/end2end/features/test_editor_bdd.py +++ b/tests/end2end/features/test_editor_bdd.py @@ -49,7 +49,7 @@ def set_up_editor_replacement(quteproc, server, tmpdir, text, replacement): @bdd.when(bdd.parsers.parse('I set up a fake editor returning "{text}"')) -def set_up_editor(quteproc, server, tmpdir, text): +def set_up_editor(quteproc, tmpdir, text): """Set up editor.command to a small python script inserting a text.""" script = tmpdir / 'script.py' script.write(textwrap.dedent(""" @@ -63,13 +63,13 @@ def set_up_editor(quteproc, server, tmpdir, text): @bdd.when(bdd.parsers.parse('I set up a fake editor returning empty text')) -def set_up_editor_empty(quteproc, server, tmpdir): +def set_up_editor_empty(quteproc, tmpdir): """Set up editor.command to a small python script inserting empty text.""" - set_up_editor(quteproc, server, tmpdir, "") + set_up_editor(quteproc, tmpdir, "") @bdd.when(bdd.parsers.parse('I set up a fake editor that waits')) -def set_up_editor_wait(quteproc, server, tmpdir): +def set_up_editor_wait(quteproc, tmpdir): """Set up editor.command to a small python script inserting a text.""" pidfile = tmpdir / 'editor_pid' script = tmpdir / 'script.py' @@ -90,7 +90,7 @@ def set_up_editor_wait(quteproc, server, tmpdir): @bdd.when(bdd.parsers.parse('I kill the waiting editor')) -def kill_editor_wait(quteproc, server, tmpdir): +def kill_editor_wait(tmpdir): """Kill the waiting editor.""" pidfile = tmpdir / 'editor_pid' pid = int(pidfile.read()) From df6ff55b7a34d543b9235bf71f0131b2598ff1c9 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Fri, 1 Dec 2017 10:51:41 -0600 Subject: [PATCH 217/415] allow pytest to default to link_pyqt link_pyqt now checks for LINK_PYQT_SKIP, allowing pytest env names like `py36` to work properly without negative conditionals in tox.ini --- scripts/link_pyqt.py | 4 ++++ tox.ini | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/link_pyqt.py b/scripts/link_pyqt.py index 57eeb9138..82f8cbac5 100644 --- a/scripts/link_pyqt.py +++ b/scripts/link_pyqt.py @@ -205,6 +205,10 @@ def main(): args = parser.parse_args() if args.tox: + #workaround for the lack of negative factors in tox.ini + if 'LINK_PYQT_SKIP' in os.environ: + print('LINK_PYQT_SKIP set, exiting...') + sys.exit(0) executable = get_tox_syspython(args.path) else: executable = sys.executable diff --git a/tox.ini b/tox.ini index de41f2e0f..2ada5ab58 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ skipsdist = true setenv = QT_QPA_PLATFORM_PLUGIN_PATH={envdir}/Lib/site-packages/PyQt5/plugins/platforms PYTEST_QT_API=pyqt5 + pyqt{,56,571,58,59}: LINK_PYQT_SKIP=true pyqt{,56,571,58,59}: QUTE_BDD_WEBENGINE=true cov: PYTEST_ADDOPTS=--cov --cov-report xml --cov-report=html --cov-report= passenv = PYTHON DISPLAY XAUTHORITY HOME USERNAME USER CI TRAVIS XDG_* QUTE_* DOCKER @@ -27,7 +28,7 @@ deps = pyqt58: PyQt5==5.8.2 pyqt59: PyQt5==5.9.2 commands = - pyqtlink: {envpython} scripts/link_pyqt.py --tox {envdir} + {envpython} scripts/link_pyqt.py --tox {envdir} {envpython} -bb -m pytest {posargs:tests} cov: {envpython} scripts/dev/check_coverage.py {posargs} From 5607cc2be8c379c002dc257b3fd9a0d606ede98d Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Fri, 1 Dec 2017 10:52:58 -0600 Subject: [PATCH 218/415] Revert "update contributing.asciidoc with -pyqtlink envs" This reverts commit 6b762037809b9057774bb88f333466d6d1ba9bd3. --- doc/contributing.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing.asciidoc b/doc/contributing.asciidoc index 6f0b1bb72..afbb752c5 100644 --- a/doc/contributing.asciidoc +++ b/doc/contributing.asciidoc @@ -97,7 +97,7 @@ unittests and several linters/checkers. Currently, the following tox environments are available: * Tests using https://www.pytest.org[pytest]: - - `py35-pyqtlink`, `py36-pyqtlink`: Run pytest for python 3.5/3.6 with the system-wide PyQt. + - `py35`, `py36`: Run pytest for python 3.5/3.6 with the system-wide PyQt. - `py36-pyqt57`, ..., `py36-pyqt59`: Run pytest with the given PyQt version (`py35-*` also works). - `py36-pyqt59-cov`: Run with coverage support (other Python/PyQt versions work too). * `flake8`: Run various linting checks via https://pypi.python.org/pypi/flake8[flake8]. From fbd325f8d1a95507b656b3111b11d87d3e20aa1b Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Fri, 1 Dec 2017 10:55:08 -0600 Subject: [PATCH 219/415] Revert "update tox env name in CI config" This reverts commit 2f231c86ac76cd47038759331b3a1b26418043e0. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e92df0b6b..65d917d73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ matrix: - os: linux env: TESTENV=py36-pyqt59-cov - os: osx - env: TESTENV=py36-pyqtlink OSX=sierra + env: TESTENV=py36 OSX=sierra osx_image: xcode8.3 language: generic # https://github.com/qutebrowser/qutebrowser/issues/2013 From 689fe963934abf26300e08c513ebec593cb5ecda Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sat, 2 Dec 2017 14:37:59 +0100 Subject: [PATCH 220/415] Improve workaround comment --- qutebrowser/completion/models/completionmodel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/completion/models/completionmodel.py b/qutebrowser/completion/models/completionmodel.py index 004848cfb..aa4422d83 100644 --- a/qutebrowser/completion/models/completionmodel.py +++ b/qutebrowser/completion/models/completionmodel.py @@ -177,7 +177,8 @@ class CompletionModel(QAbstractItemModel): pattern: The filter pattern to set. """ log.completion.debug("Setting completion pattern '{}'".format(pattern)) - # layoutChanged is broken in pyqt-5.7.1, so we must use metaObject + # WORKAROUND: + # layoutChanged is broken in PyQt 5.7.1, so we must use metaObject # https://www.riverbankcomputing.com/pipermail/pyqt/2017-January/038483.html self.metaObject().invokeMethod(self, "layoutAboutToBeChanged") for cat in self._categories: From a41db970e8f916a037a85feca94be50cdc35c109 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sat, 2 Dec 2017 14:42:14 +0100 Subject: [PATCH 221/415] Update changelog [ci skip] --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 087619870..f972c6ca2 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -88,6 +88,7 @@ Changed - `:jseval --file` now searches relative paths in a js/ subdir in qutebrowser's data dir, e.g. `~/.local/share/qutebrowser/js`. - The current/default bindings are now shown in the :bind completion. +- Empty categories are now hidden in the `:open` completion. Fixed ~~~~~ From f1f573d651c14b57d502a12dcf07005fae300848 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sat, 2 Dec 2017 14:48:31 +0100 Subject: [PATCH 222/415] Update changelog --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index f972c6ca2..34ee4ff7c 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -106,6 +106,7 @@ Fixed - Fixed `gU` (`:navigate up`) on `qute://help` and webservers not handling `..` in a URL. - Using e.g. `-s backend webkit` to set the backend now works correctly. +- Fixed crash when closing the tab an external editor was opened in. Deprecated ~~~~~~~~~~ From ce46b30a1e21ef5441dd9c4640a2f9f93d82b9a9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sat, 2 Dec 2017 14:49:06 +0100 Subject: [PATCH 223/415] Show error instead of warning --- qutebrowser/browser/commands.py | 2 +- tests/end2end/features/editor.feature | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 781dfeb24..0de903004 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1645,7 +1645,7 @@ class CommandDispatcher: try: elem.set_value(text) except webelem.OrphanedError as e: - message.warning('Edited element vanished') + message.error('Edited element vanished') except webelem.Error as e: raise cmdexc.CommandError(str(e)) diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 4647eb08c..21f3df425 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -126,7 +126,7 @@ Feature: Opening external editors And I set tabs.last_close to blank And I run :tab-close And I kill the waiting editor - Then the warning "Edited element vanished" should be shown + Then the error "Edited element vanished" should be shown @qtwebengine_todo: Caret mode is not implemented yet Scenario: Spawning an editor in caret mode From b91a39be220c86fa2659e6cfa73b16c54a8686f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sat, 2 Dec 2017 19:22:14 +0100 Subject: [PATCH 224/415] Run shellcheck on Travis CI --- .travis.yml | 3 +++ scripts/dev/ci/travis_run.sh | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index 65d917d73..37387b620 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,9 @@ matrix: language: node_js python: null node_js: "lts/*" + - os: linux + env: TESTENV=shellcheck + services: docker fast_finish: true cache: diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index 2a5424fb9..70c41b0c3 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -14,6 +14,16 @@ elif [[ $TESTENV == eslint ]]; then # travis env cd qutebrowser/javascript || exit 1 eslint --color --report-unused-disable-directives . +elif [[ $TESTENV == shellcheck ]]; then + dev_scripts=$( find scripts/dev/ -name '*.sh' -print0 | xargs -0 ) + # false positive: we are using 'find -exec +' + # shellcheck disable=SC2038 + userscripts=$( find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + | xargs ) + IFS=" " read -r -a scripts <<< "$dev_scripts $userscripts" + docker run \ + -v "$PWD:/outside" \ + -w /outside \ + koalaman/shellcheck:stable "${scripts[@]}" else args=() [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb') From 595a53ad3b29570bed35764b0b2cbfdda9af2dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sat, 2 Dec 2017 19:23:55 +0100 Subject: [PATCH 225/415] Apply patch from #1697 --- misc/userscripts/open_download | 3 ++- misc/userscripts/password_fill | 6 +++--- misc/userscripts/qutedmenu | 12 +++--------- misc/userscripts/taskadd | 6 +++--- scripts/dev/ci/travis_install.sh | 6 +++--- scripts/dev/quit_segfault_test.sh | 10 ++++------ 6 files changed, 18 insertions(+), 25 deletions(-) diff --git a/misc/userscripts/open_download b/misc/userscripts/open_download index 6c1213b65..85ea7f849 100755 --- a/misc/userscripts/open_download +++ b/misc/userscripts/open_download @@ -76,6 +76,7 @@ crop-first-column() { ls-files() { # add the slash at the end of the download dir enforces to follow the # symlink, if the DOWNLOAD_DIR itself is a symlink + # shellcheck disable=SC2010 ls -Q --quoting-style escape -h -o -1 -A -t "${DOWNLOAD_DIR}/" \ | grep '^[-]' \ | cut -d' ' -f3- \ @@ -94,7 +95,7 @@ fi line=$(printf "%s\n" "${entries[@]}" \ | crop-first-column 55 \ | column -s $'\t' -t \ - | $ROFI_CMD "${rofi_default_args[@]}" $ROFI_ARGS) || true + | $ROFI_CMD "${rofi_default_args[@]}" "$ROFI_ARGS") || true if [ -z "$line" ]; then exit 0 fi diff --git a/misc/userscripts/password_fill b/misc/userscripts/password_fill index af394ac2c..f2157f190 100755 --- a/misc/userscripts/password_fill +++ b/misc/userscripts/password_fill @@ -236,7 +236,7 @@ pass_backend() { if ((match_line)) ; then # add entries with matching URL-tag while read -r -d "" passfile ; do - if $GPG "${GPG_OPTS}" -d "$passfile" \ + if $GPG "${GPG_OPTS[@]}" -d "$passfile" \ | grep --max-count=1 -iE "${match_line_pattern}${url}" > /dev/null then passfile="${passfile#$PREFIX}" @@ -269,7 +269,7 @@ pass_backend() { break fi fi - done < <($GPG "${GPG_OPTS}" -d "$path" ) + done < <($GPG "${GPG_OPTS[@]}" -d "$path" ) } } # ======================================================= @@ -283,7 +283,7 @@ secret_backend() { query_entries() { local domain="$1" while read -r line ; do - if [[ "$line" =~ "attribute.username = " ]] ; then + if [[ "$line" == "attribute.username ="* ]] ; then files+=("$domain ${line#${BASH_REMATCH[0]}}") fi done < <( secret-tool search --unlock --all domain "$domain" 2>&1 ) diff --git a/misc/userscripts/qutedmenu b/misc/userscripts/qutedmenu index 3f8b13514..b753c6d5b 100755 --- a/misc/userscripts/qutedmenu +++ b/misc/userscripts/qutedmenu @@ -35,17 +35,11 @@ get_selection() { # Main # https://github.com/halfwit/dotfiles/blob/master/.config/dmenu/font -if [[ -s $confdir/dmenu/font ]]; then - read -r font < "$confdir"/dmenu/font -fi +[[ -s $confdir/dmenu/font ]] && read -r font < "$confdir"/dmenu/font -if [[ $font ]]; then - opts+=(-fn "$font") -fi +[[ $font ]] && opts+=(-fn "$font") -if [[ -s $optsfile ]]; then - source "$optsfile" -fi +[[ -s $optsfile ]] && source "$optsfile" url=$(get_selection) url=${url/*http/http} diff --git a/misc/userscripts/taskadd b/misc/userscripts/taskadd index 6add71c68..9c70fb978 100755 --- a/misc/userscripts/taskadd +++ b/misc/userscripts/taskadd @@ -25,12 +25,12 @@ [[ $QUTE_MODE == 'hints' ]] && title=$QUTE_SELECTED_TEXT || title=$QUTE_TITLE # try to add the task and grab the output -msg="$(task add $title $@ 2>&1)" +msg="$(task add "$title" "$*" 2>&1)" if [[ $? == 0 ]]; then # annotate the new task with the url, send the output back to the browser task +LATEST annotate "$QUTE_URL" - echo "message-info '$msg'" >> $QUTE_FIFO + echo "message-info '$msg'" >> "$QUTE_FIFO" else - echo "message-error '$msg'" >> $QUTE_FIFO + echo "message-error '$msg'" >> "$QUTE_FIFO" fi diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index 4c599aac6..45bb5fe1e 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -27,17 +27,17 @@ travis_retry() { local count=1 while (( count < 3 )); do if (( result != 0 )); then - echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2 + echo -e "\n${ANSI_RED}The command \"$*\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2 fi "$@" result=$? (( result == 0 )) && break - count=$(($count + 1)) + count=$(( count + 1 )) sleep 1 done if (( count > 3 )); then - echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2 + echo -e "\n${ANSI_RED}The command \"$*\" failed 3 times.${ANSI_RESET}\n" >&2 fi return $result diff --git a/scripts/dev/quit_segfault_test.sh b/scripts/dev/quit_segfault_test.sh index 655eb262a..389f125b9 100755 --- a/scripts/dev/quit_segfault_test.sh +++ b/scripts/dev/quit_segfault_test.sh @@ -1,14 +1,12 @@ -#!/bin/bash +#!/usr/bin/env bash -if [[ $PWD == */scripts ]]; then - cd .. -fi +[[ $PWD == */scripts ]] && cd .. echo > crash.log while :; do exit=0 - while (( $exit == 0)); do - duration=$(($RANDOM%10000)) + while (( exit == 0 )); do + duration=$(( RANDOM % 10000 )) python3 -m qutebrowser --debug ":later $duration quit" http://www.heise.de/ exit=$? done From 31710b7045923dc984035984565f23b76a4d5ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sat, 2 Dec 2017 19:31:52 +0100 Subject: [PATCH 226/415] Trivial fixes for shellcheck warnings --- misc/userscripts/cast | 4 ++-- misc/userscripts/dmenu_qutebrowser | 2 +- misc/userscripts/format_json | 4 ++-- misc/userscripts/open_download | 2 +- misc/userscripts/password_fill | 5 +++-- misc/userscripts/qutedmenu | 1 + misc/userscripts/rss | 10 +++++----- scripts/dev/ci/travis_backtrace.sh | 2 +- scripts/dev/ci/travis_install.sh | 8 ++++---- scripts/dev/download_release.sh | 6 +++--- 10 files changed, 23 insertions(+), 21 deletions(-) diff --git a/misc/userscripts/cast b/misc/userscripts/cast index da68297d8..f7b64df70 100755 --- a/misc/userscripts/cast +++ b/misc/userscripts/cast @@ -144,7 +144,7 @@ fi pkill -f "${program_}" # start youtube download in stream mode (-o -) into temporary file -youtube-dl -qo - "$1" > ${file_to_cast} & +youtube-dl -qo - "$1" > "${file_to_cast}" & ytdl_pid=$! msg info "Casting $1" >> "$QUTE_FIFO" @@ -153,4 +153,4 @@ tail -F "${file_to_cast}" | ${program_} - # cleanup remaining background process and file on disk kill ${ytdl_pid} -rm -rf ${tmpdir} +rm -rf "${tmpdir}" diff --git a/misc/userscripts/dmenu_qutebrowser b/misc/userscripts/dmenu_qutebrowser index 9c809d5ad..82e6d2f18 100755 --- a/misc/userscripts/dmenu_qutebrowser +++ b/misc/userscripts/dmenu_qutebrowser @@ -41,7 +41,7 @@ [ -z "$QUTE_URL" ] && QUTE_URL='http://google.com' url=$(echo "$QUTE_URL" | cat - "$QUTE_CONFIG_DIR/quickmarks" "$QUTE_DATA_DIR/history" | dmenu -l 15 -p qutebrowser) -url=$(echo "$url" | sed -E 's/[^ ]+ +//g' | egrep "https?:" || echo "$url") +url=$(echo "$url" | sed -E 's/[^ ]+ +//g' | grep -E "https?:" || echo "$url") [ -z "${url// }" ] && exit diff --git a/misc/userscripts/format_json b/misc/userscripts/format_json index f756850f1..42d8dea14 100755 --- a/misc/userscripts/format_json +++ b/misc/userscripts/format_json @@ -35,12 +35,12 @@ FILE_SIZE=$(ls -s --block-size=1048576 "$QUTE_TEXT" | cut -d' ' -f1) # use pygments to pretty-up the json (syntax highlight) if file is less than 10MB if [ "$FILE_SIZE" -lt "10" ]; then - FORMATTED_JSON="$(echo "$FORMATTED_JSON" | pygmentize -l json -f html -O full,style=$STYLE)" + FORMATTED_JSON="$(echo "$FORMATTED_JSON" | pygmentize -l json -f html -O full,style="$STYLE")" fi # create a temp file and write the formatted json to that file TEMP_FILE="$(mktemp --suffix '.html')" -echo "$FORMATTED_JSON" > $TEMP_FILE +echo "$FORMATTED_JSON" > "$TEMP_FILE" # send the command to qutebrowser to open the new file containing the formatted json diff --git a/misc/userscripts/open_download b/misc/userscripts/open_download index 85ea7f849..ecc1d7209 100755 --- a/misc/userscripts/open_download +++ b/misc/userscripts/open_download @@ -92,7 +92,7 @@ if [ "${#entries[@]}" -eq 0 ] ; then die "Download directory »${DOWNLOAD_DIR}« empty" fi -line=$(printf "%s\n" "${entries[@]}" \ +line=$(printf '%s\n' "${entries[@]}" \ | crop-first-column 55 \ | column -s $'\t' -t \ | $ROFI_CMD "${rofi_default_args[@]}" "$ROFI_ARGS") || true diff --git a/misc/userscripts/password_fill b/misc/userscripts/password_fill index f2157f190..22497d987 100755 --- a/misc/userscripts/password_fill +++ b/misc/userscripts/password_fill @@ -178,7 +178,7 @@ choose_entry_menu() { if [ "$nr" -eq 1 ] && ! ((menu_if_one_entry)) ; then file="${files[0]}" else - file=$( printf "%s\n" "${files[@]}" | "${MENU_COMMAND[@]}" ) + file=$( printf '%s\n' "${files[@]}" | "${MENU_COMMAND[@]}" ) fi } @@ -303,6 +303,7 @@ pass_backend QUTE_CONFIG_DIR=${QUTE_CONFIG_DIR:-${XDG_CONFIG_HOME:-$HOME/.config}/qutebrowser/} PWFILL_CONFIG=${PWFILL_CONFIG:-${QUTE_CONFIG_DIR}/password_fill_rc} if [ -f "$PWFILL_CONFIG" ] ; then + # shellcheck source=/dev/null source "$PWFILL_CONFIG" fi init @@ -311,7 +312,7 @@ simplify_url "$QUTE_URL" query_entries "${simple_url}" no_entries_found # remove duplicates -mapfile -t files < <(printf "%s\n" "${files[@]}" | sort | uniq ) +mapfile -t files < <(printf '%s\n' "${files[@]}" | sort | uniq ) choose_entry if [ -z "$file" ] ; then # choose_entry didn't want any of these entries diff --git a/misc/userscripts/qutedmenu b/misc/userscripts/qutedmenu index b753c6d5b..de1b8d641 100755 --- a/misc/userscripts/qutedmenu +++ b/misc/userscripts/qutedmenu @@ -39,6 +39,7 @@ get_selection() { [[ $font ]] && opts+=(-fn "$font") +# shellcheck source=/dev/null [[ -s $optsfile ]] && source "$optsfile" url=$(get_selection) diff --git a/misc/userscripts/rss b/misc/userscripts/rss index 222d990a2..6680a259e 100755 --- a/misc/userscripts/rss +++ b/misc/userscripts/rss @@ -32,7 +32,7 @@ add_feed () { if grep -Fq "$1" "feeds"; then notice "$1 is saved already." else - printf "%s\n" "$1" >> "feeds" + printf '%s\n' "$1" >> "feeds" fi } @@ -57,7 +57,7 @@ notice () { # Update a database of a feed and open new URLs read_items () { - cd read_urls + cd read_urls || return feed_file="$(echo "$1" | tr -d /)" feed_temp_file="$(mktemp "$feed_file.tmp.XXXXXXXXXX")" feed_new_items="$(mktemp "$feed_file.new.XXXXXXXXXX")" @@ -75,7 +75,7 @@ read_items () { cat "$feed_new_items" >> "$feed_file" sort -o "$feed_file" "$feed_file" rm "$feed_temp_file" "$feed_new_items" - fi | while read item; do + fi | while read -r item; do echo "open -t $item" > "$QUTE_FIFO" done } @@ -85,7 +85,7 @@ if [ ! -d "$config_dir/read_urls" ]; then mkdir -p "$config_dir/read_urls" fi -cd "$config_dir" +cd "$config_dir" || exit if [ $# != 0 ]; then for arg in "$@"; do @@ -115,7 +115,7 @@ if < /dev/null grep --help 2>&1 | grep -q -- -a; then text_only="-a" fi -while read feed_url; do +while read -r feed_url; do read_items "$feed_url" & done < "$config_dir/feeds" diff --git a/scripts/dev/ci/travis_backtrace.sh b/scripts/dev/ci/travis_backtrace.sh index c94d1ff06..4027f7c10 100644 --- a/scripts/dev/ci/travis_backtrace.sh +++ b/scripts/dev/ci/travis_backtrace.sh @@ -6,7 +6,7 @@ case $TESTENV in py3*-pyqt*) - exe=$(readlink -f .tox/$TESTENV/bin/python) + exe=$(readlink -f ".tox/$TESTENV/bin/python") full= ;; *) diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index 45bb5fe1e..18ac44048 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -21,13 +21,13 @@ # Stolen from https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/templates/header.sh # and adjusted to use ((...)) travis_retry() { - local ANSI_RED="\033[31;1m" - local ANSI_RESET="\033[0m" + local ANSI_RED='\033[31;1m' + local ANSI_RESET='\033[0m' local result=0 local count=1 while (( count < 3 )); do if (( result != 0 )); then - echo -e "\n${ANSI_RED}The command \"$*\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2 + echo -e "\\n${ANSI_RED}The command \"$*\" failed. Retrying, $count of 3.${ANSI_RESET}\\n" >&2 fi "$@" result=$? @@ -37,7 +37,7 @@ travis_retry() { done if (( count > 3 )); then - echo -e "\n${ANSI_RED}The command \"$*\" failed 3 times.${ANSI_RESET}\n" >&2 + echo -e "\\n${ANSI_RED}The command \"$*\" failed 3 times.${ANSI_RESET}\\n" >&2 fi return $result diff --git a/scripts/dev/download_release.sh b/scripts/dev/download_release.sh index 7ec4d9159..66242ec8e 100644 --- a/scripts/dev/download_release.sh +++ b/scripts/dev/download_release.sh @@ -11,7 +11,7 @@ if [[ $# != 1 ]]; then exit 1 fi -cd "$tmpdir" +cd "$tmpdir" || exit mkdir windows base="https://github.com/qutebrowser/qutebrowser/releases/download/v$1" @@ -21,13 +21,13 @@ wget "$base/qutebrowser-$1.tar.gz.asc" || exit 1 wget "$base/qutebrowser-$1.dmg" || exit 1 wget "$base/qutebrowser_${1}-1_all.deb" || exit 1 -cd windows +cd windows || exit wget "$base/qutebrowser-${1}-amd64.msi" || exit 1 wget "$base/qutebrowser-${1}-win32.msi" || exit 1 wget "$base/qutebrowser-${1}-windows-standalone-amd64.zip" || exit 1 wget "$base/qutebrowser-${1}-windows-standalone-win32.zip" || exit 1 dest="/srv/http/qutebrowser/releases/v$1" -cd "$oldpwd" +cd "$oldpwd" || exit sudo mv "$tmpdir" "$dest" sudo chown -R http:http "$dest" From dd589f180bdd97670673957c43d4dfccd2ae6054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sat, 2 Dec 2017 19:38:02 +0100 Subject: [PATCH 227/415] Fix remaining shellcheck warnings --- misc/userscripts/format_json | 7 +++---- misc/userscripts/password_fill | 2 +- misc/userscripts/taskadd | 4 +--- misc/userscripts/view_in_mpv | 2 +- scripts/dev/ci/travis_backtrace.sh | 2 +- scripts/dev/download_release.sh | 23 ++++++++++++----------- 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/misc/userscripts/format_json b/misc/userscripts/format_json index 42d8dea14..10ff23e53 100755 --- a/misc/userscripts/format_json +++ b/misc/userscripts/format_json @@ -22,15 +22,14 @@ # default style to monokai if none is provided STYLE=${1:-monokai} # format json using jq -FORMATTED_JSON="$(cat "$QUTE_TEXT" | jq '.')" - -# if jq command failed or formatted json is empty, assume failure and terminate -if [ $? -ne 0 ] || [ -z "$FORMATTED_JSON" ]; then +if ! FORMATTED_JSON="$(jq . "$QUTE_TEXT")" || [ -z "$FORMATTED_JSON" ]; then echo "Invalid json, aborting..." exit 1 fi # calculate the filesize of the json document +# parsing the output of ls should be fine in this case since we set the block size +# shellcheck disable=SC2012 FILE_SIZE=$(ls -s --block-size=1048576 "$QUTE_TEXT" | cut -d' ' -f1) # use pygments to pretty-up the json (syntax highlight) if file is less than 10MB diff --git a/misc/userscripts/password_fill b/misc/userscripts/password_fill index 22497d987..5f30a6bf6 100755 --- a/misc/userscripts/password_fill +++ b/misc/userscripts/password_fill @@ -64,7 +64,7 @@ die() { javascript_escape() { # print the first argument in an escaped way, such that it can safely # be used within javascripts double quotes - sed "s,[\\\'\"],\\\&,g" <<< "$1" + sed "s,[\\\\'\"],\\\\&,g" <<< "$1" } # ======================================================= # diff --git a/misc/userscripts/taskadd b/misc/userscripts/taskadd index 9c70fb978..b1ded245c 100755 --- a/misc/userscripts/taskadd +++ b/misc/userscripts/taskadd @@ -25,9 +25,7 @@ [[ $QUTE_MODE == 'hints' ]] && title=$QUTE_SELECTED_TEXT || title=$QUTE_TITLE # try to add the task and grab the output -msg="$(task add "$title" "$*" 2>&1)" - -if [[ $? == 0 ]]; then +if msg="$(task add "$title" "$*" 2>&1)"; then # annotate the new task with the url, send the output back to the browser task +LATEST annotate "$QUTE_URL" echo "message-info '$msg'" >> "$QUTE_FIFO" diff --git a/misc/userscripts/view_in_mpv b/misc/userscripts/view_in_mpv index 9eb6ff7c6..f465fc4e4 100755 --- a/misc/userscripts/view_in_mpv +++ b/misc/userscripts/view_in_mpv @@ -50,7 +50,7 @@ msg() { MPV_COMMAND=${MPV_COMMAND:-mpv} # Warning: spaces in single flags are not supported MPV_FLAGS=${MPV_FLAGS:- --force-window --no-terminal --keep-open=yes --ytdl --ytdl-raw-options=yes-playlist=} -video_command=( "$MPV_COMMAND" $MPV_FLAGS ) +IFS=" " read -r -a video_command <<< "$MPV_COMMAND $MPV_FLAGS" js() { cat < Date: Sun, 3 Dec 2017 09:29:38 +0100 Subject: [PATCH 228/415] Use koalaman/shellcheck:latest --- scripts/dev/ci/travis_run.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index 70c41b0c3..f4960b808 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -16,14 +16,12 @@ elif [[ $TESTENV == eslint ]]; then eslint --color --report-unused-disable-directives . elif [[ $TESTENV == shellcheck ]]; then dev_scripts=$( find scripts/dev/ -name '*.sh' -print0 | xargs -0 ) - # false positive: we are using 'find -exec +' - # shellcheck disable=SC2038 userscripts=$( find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + | xargs ) IFS=" " read -r -a scripts <<< "$dev_scripts $userscripts" docker run \ -v "$PWD:/outside" \ -w /outside \ - koalaman/shellcheck:stable "${scripts[@]}" + koalaman/shellcheck:latest "${scripts[@]}" else args=() [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb') From 22e4a800a1c6080d29d41eeda229bc67b7bf314b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sun, 3 Dec 2017 10:50:54 +0100 Subject: [PATCH 229/415] Refactor format_json userscript to not parse 'ls' output The script now also works under MacOS --- misc/userscripts/format_json | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/misc/userscripts/format_json b/misc/userscripts/format_json index 10ff23e53..0d476b327 100755 --- a/misc/userscripts/format_json +++ b/misc/userscripts/format_json @@ -1,4 +1,5 @@ #!/bin/sh +set -euo pipefail # # Behavior: # Userscript for qutebrowser which will take the raw JSON text of the current @@ -19,28 +20,23 @@ # # Bryan Gilbert, 2017 +# do not run pygmentize on files larger than this amount of bytes +MAX_SIZE_PRETTIFY=10485760 # 10 MB # default style to monokai if none is provided STYLE=${1:-monokai} -# format json using jq -if ! FORMATTED_JSON="$(jq . "$QUTE_TEXT")" || [ -z "$FORMATTED_JSON" ]; then - echo "Invalid json, aborting..." - exit 1 + +TEMP_FILE="$(mktemp)" +jq . "$QUTE_TEXT" >"$TEMP_FILE" + +# try GNU stat first and then OSX stat if the former fails +FILE_SIZE=$( + stat --printf="%s" "$TEMP_FILE" 2>/dev/null || + stat -f%z "$TEMP_FILE" 2>/dev/null +) +if [ "$FILE_SIZE" -lt "$MAX_SIZE_PRETTIFY" ]; then + pygmentize -l json -f html -O full,style="$STYLE" <"$TEMP_FILE" >"${TEMP_FILE}_" + mv -f "${TEMP_FILE}_" "$TEMP_FILE" fi -# calculate the filesize of the json document -# parsing the output of ls should be fine in this case since we set the block size -# shellcheck disable=SC2012 -FILE_SIZE=$(ls -s --block-size=1048576 "$QUTE_TEXT" | cut -d' ' -f1) - -# use pygments to pretty-up the json (syntax highlight) if file is less than 10MB -if [ "$FILE_SIZE" -lt "10" ]; then - FORMATTED_JSON="$(echo "$FORMATTED_JSON" | pygmentize -l json -f html -O full,style="$STYLE")" -fi - -# create a temp file and write the formatted json to that file -TEMP_FILE="$(mktemp --suffix '.html')" -echo "$FORMATTED_JSON" > "$TEMP_FILE" - - # send the command to qutebrowser to open the new file containing the formatted json echo "open -t file://$TEMP_FILE" >> "$QUTE_FIFO" From 59c9a2b243204f4600018c12ccdad3fa40404071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sun, 3 Dec 2017 11:30:59 +0100 Subject: [PATCH 230/415] Ignore shellcheck false positive --- scripts/dev/ci/travis_run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index f4960b808..74b8ae63f 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -16,6 +16,8 @@ elif [[ $TESTENV == eslint ]]; then eslint --color --report-unused-disable-directives . elif [[ $TESTENV == shellcheck ]]; then dev_scripts=$( find scripts/dev/ -name '*.sh' -print0 | xargs -0 ) + # false positive: we are using 'find -exec +' + # shellcheck disable=SC2038 userscripts=$( find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + | xargs ) IFS=" " read -r -a scripts <<< "$dev_scripts $userscripts" docker run \ From f07301cfb5f31e02b26e1865a8d71389f002c0d2 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 3 Dec 2017 12:48:29 +0100 Subject: [PATCH 231/415] Revert "Restart correctly after reporting crash." This reverts commit 7001f068b3ef29dce63feab19ee008d8c7e5ca27. --- qutebrowser/misc/crashdialog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/misc/crashdialog.py b/qutebrowser/misc/crashdialog.py index 3d50e5f29..9919386c4 100644 --- a/qutebrowser/misc/crashdialog.py +++ b/qutebrowser/misc/crashdialog.py @@ -348,7 +348,7 @@ class _CrashDialog(QDialog): "but you're currently running v{} - please " "update!".format(newest, qutebrowser.__version__)) text = '

'.join(lines) - self.finish() + self.hide() msgbox.information(self, "Report successfully sent!", text, on_finished=self.finish, plain_text=False) @@ -365,7 +365,7 @@ class _CrashDialog(QDialog): "qutebrowser.org " "by yourself.".format(msg)) text = '

'.join(lines) - self.finish() + self.hide() msgbox.information(self, "Report successfully sent!", text, on_finished=self.finish, plain_text=False) From 97054ca35de48294ff549b82d2d586bd7c7b28ce Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 3 Dec 2017 13:04:08 +0100 Subject: [PATCH 232/415] Don't hide report dialog early It looks like hiding it already causes it to be accepted. Fixes #1128 --- qutebrowser/misc/crashdialog.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qutebrowser/misc/crashdialog.py b/qutebrowser/misc/crashdialog.py index 9919386c4..54827211a 100644 --- a/qutebrowser/misc/crashdialog.py +++ b/qutebrowser/misc/crashdialog.py @@ -348,7 +348,6 @@ class _CrashDialog(QDialog): "but you're currently running v{} - please " "update!".format(newest, qutebrowser.__version__)) text = '

'.join(lines) - self.hide() msgbox.information(self, "Report successfully sent!", text, on_finished=self.finish, plain_text=False) @@ -365,7 +364,6 @@ class _CrashDialog(QDialog): "qutebrowser.org " "by yourself.".format(msg)) text = '

'.join(lines) - self.hide() msgbox.information(self, "Report successfully sent!", text, on_finished=self.finish, plain_text=False) From b610563e7fed16f6403925fb5d17eb2f9f9a70a7 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 3 Dec 2017 07:32:55 -0500 Subject: [PATCH 233/415] Don't show current window for :tab-give/:tab-take. Resolves #3144. --- qutebrowser/browser/commands.py | 2 +- qutebrowser/completion/completer.py | 8 +++-- qutebrowser/completion/models/miscmodels.py | 30 +++++++++++++++--- qutebrowser/mainwindow/mainwindow.py | 2 +- tests/unit/completion/test_completer.py | 2 +- tests/unit/completion/test_models.py | 35 ++++++++++++++++++--- 6 files changed, 65 insertions(+), 14 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 0de903004..bced4daf4 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -518,7 +518,7 @@ class CommandDispatcher: return newtab @cmdutils.register(instance='command-dispatcher', scope='window') - @cmdutils.argument('index', completion=miscmodels.buffer) + @cmdutils.argument('index', completion=miscmodels.other_buffer) def tab_take(self, index): """Take a tab from another window. diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index bc0e4991f..15c5d7f14 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -35,6 +35,7 @@ class CompletionInfo: config = attr.ib() keyconf = attr.ib() + win_id = attr.ib() class Completer(QObject): @@ -43,6 +44,7 @@ class Completer(QObject): Attributes: _cmd: The statusbar Command object this completer belongs to. + _win_id: The id of the window that owns this object. _timer: The timer used to trigger the completion update. _last_cursor_pos: The old cursor position so we avoid double completion updates. @@ -50,9 +52,10 @@ class Completer(QObject): _last_completion_func: The completion function used for the last text. """ - def __init__(self, cmd, parent=None): + def __init__(self, cmd, win_id, parent=None): super().__init__(parent) self._cmd = cmd + self._win_id = win_id self._timer = QTimer() self._timer.setSingleShot(True) self._timer.setInterval(0) @@ -250,7 +253,8 @@ class Completer(QObject): with debug.log_time(log.completion, 'Starting {} completion'.format(func.__name__)): info = CompletionInfo(config=config.instance, - keyconf=config.key_instance) + keyconf=config.key_instance, + win_id=self._win_id) model = func(*args, info=info) with debug.log_time(log.completion, 'Set completion model'): completion.set_model(model) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 28ff0ddac..6175dcfb4 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -94,10 +94,10 @@ def session(*, info=None): # pylint: disable=unused-argument return model -def buffer(*, info=None): # pylint: disable=unused-argument - """A model to complete on open tabs across all windows. - - Used for switching the buffer command. +def _buffer(skip_win_id=None): + """Helper to get the completion model for buffer/other_buffer. + Args: + skip_win_id: The id of the window to skip, or None to include all. """ def delete_buffer(data): """Close the selected tab.""" @@ -109,6 +109,8 @@ def buffer(*, info=None): # pylint: disable=unused-argument model = completionmodel.CompletionModel(column_widths=(6, 40, 54)) for win_id in objreg.window_registry: + if skip_win_id and win_id == skip_win_id: + continue tabbed_browser = objreg.get('tabbed-browser', scope='window', window=win_id) if tabbed_browser.shutting_down: @@ -126,13 +128,31 @@ def buffer(*, info=None): # pylint: disable=unused-argument return model -def window(*, info=None): # pylint: disable=unused-argument +def buffer(*, info=None): # pylint: disable=unused-argument + """A model to complete on open tabs across all windows. + + Used for switching the buffer command. + """ + return _buffer() + + +def other_buffer(*, info): + """A model to complete on open tabs across all windows except the current. + + Used for the tab-take command. + """ + return _buffer(skip_win_id=info.win_id) + + +def window(*, info): """A model to complete on all open windows.""" model = completionmodel.CompletionModel(column_widths=(6, 30, 64)) windows = [] for win_id in objreg.window_registry: + if win_id == info.win_id: + continue tabbed_browser = objreg.get('tabbed-browser', scope='window', window=win_id) tab_titles = (tab.title() for tab in tabbed_browser.widgets()) diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index bf95d3e6a..5acec2384 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -320,7 +320,7 @@ class MainWindow(QWidget): def _init_completion(self): self._completion = completionwidget.CompletionView(self.win_id, self) cmd = objreg.get('status-command', scope='window', window=self.win_id) - completer_obj = completer.Completer(cmd, self._completion) + completer_obj = completer.Completer(cmd, self.win_id, self._completion) self._completion.selection_changed.connect( completer_obj.on_selection_changed) objreg.register('completion', self._completion, scope='window', diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index a32241621..0e8aabb93 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -65,7 +65,7 @@ def completer_obj(qtbot, status_command_stub, config_stub, monkeypatch, stubs, """Create the completer used for testing.""" monkeypatch.setattr(completer, 'QTimer', stubs.InstaTimer) config_stub.val.completion.show = 'auto' - return completer.Completer(status_command_stub, completion_widget_stub) + return completer.Completer(status_command_stub, 0, completion_widget_stub) @pytest.fixture(autouse=True) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 9c767c102..8879f3201 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -191,7 +191,8 @@ def web_history_populated(web_history): @pytest.fixture def info(config_stub, key_config_stub): return completer.CompletionInfo(config=config_stub, - keyconf=key_config_stub) + keyconf=key_config_stub, + win_id=0) def test_command_completion(qtmodeltester, cmdutils_stub, configdata_stub, @@ -581,7 +582,33 @@ def test_tab_completion_delete(qtmodeltester, fake_web_tab, app_stub, QUrl('https://duckduckgo.com')] -def test_window_completion(qtmodeltester, fake_web_tab, tabbed_browser_stubs): +def test_other_buffer_completion(qtmodeltester, fake_web_tab, app_stub, + win_registry, tabbed_browser_stubs, info): + tabbed_browser_stubs[0].tabs = [ + fake_web_tab(QUrl('https://github.com'), 'GitHub', 0), + fake_web_tab(QUrl('https://wikipedia.org'), 'Wikipedia', 1), + fake_web_tab(QUrl('https://duckduckgo.com'), 'DuckDuckGo', 2), + ] + tabbed_browser_stubs[1].tabs = [ + fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0), + ] + info.win_id = 1 + model = miscmodels.other_buffer(info=info) + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + _check_completions(model, { + '0': [ + ('0/1', 'https://github.com', 'GitHub'), + ('0/2', 'https://wikipedia.org', 'Wikipedia'), + ('0/3', 'https://duckduckgo.com', 'DuckDuckGo') + ], + }) + + +def test_window_completion(qtmodeltester, fake_web_tab, tabbed_browser_stubs, + info): tabbed_browser_stubs[0].tabs = [ fake_web_tab(QUrl('https://github.com'), 'GitHub', 0), fake_web_tab(QUrl('https://wikipedia.org'), 'Wikipedia', 1), @@ -591,7 +618,8 @@ def test_window_completion(qtmodeltester, fake_web_tab, tabbed_browser_stubs): fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0) ] - model = miscmodels.window() + info.win_id = 1 + model = miscmodels.window(info=info) model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -600,7 +628,6 @@ def test_window_completion(qtmodeltester, fake_web_tab, tabbed_browser_stubs): 'Windows': [ ('0', 'window title - qutebrowser', 'GitHub, Wikipedia, DuckDuckGo'), - ('1', 'window title - qutebrowser', 'ArchWiki') ] }) From 38b2d42b4019d31febf298667041c6bbf6e6cd9e Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Sun, 3 Dec 2017 15:09:47 -0600 Subject: [PATCH 234/415] cleanup PYTEST_ADDOPTS for pytest subprocess See https://github.com/qutebrowser/qutebrowser/pull/3349 --- tests/unit/scripts/test_check_coverage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/scripts/test_check_coverage.py b/tests/unit/scripts/test_check_coverage.py index 6b18568c5..d7183111f 100644 --- a/tests/unit/scripts/test_check_coverage.py +++ b/tests/unit/scripts/test_check_coverage.py @@ -50,6 +50,7 @@ class CovtestHelper: def run(self): """Run pytest with coverage for the given module.py.""" coveragerc = str(self._testdir.tmpdir / 'coveragerc') + self._monkeypatch.delenv('PYTEST_ADDOPTS', raising=False) return self._testdir.runpytest('--cov=module', '--cov-config={}'.format(coveragerc), '--cov-report=xml', From a137a29ccea17fa0228e339182b656ed63090e27 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 3 Dec 2017 22:32:17 +0100 Subject: [PATCH 235/415] Style improvements This adds a blank line and makes Completer arguments keyword-only to make their meaning more clear. --- qutebrowser/completion/completer.py | 2 +- qutebrowser/completion/models/miscmodels.py | 1 + qutebrowser/mainwindow/mainwindow.py | 3 ++- tests/unit/completion/test_completer.py | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 15c5d7f14..edb1aca97 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -52,7 +52,7 @@ class Completer(QObject): _last_completion_func: The completion function used for the last text. """ - def __init__(self, cmd, win_id, parent=None): + def __init__(self, *, cmd, win_id, parent=None): super().__init__(parent) self._cmd = cmd self._win_id = win_id diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 6175dcfb4..b1e599798 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -96,6 +96,7 @@ def session(*, info=None): # pylint: disable=unused-argument def _buffer(skip_win_id=None): """Helper to get the completion model for buffer/other_buffer. + Args: skip_win_id: The id of the window to skip, or None to include all. """ diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index 5acec2384..7c35f0529 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -320,7 +320,8 @@ class MainWindow(QWidget): def _init_completion(self): self._completion = completionwidget.CompletionView(self.win_id, self) cmd = objreg.get('status-command', scope='window', window=self.win_id) - completer_obj = completer.Completer(cmd, self.win_id, self._completion) + completer_obj = completer.Completer(cmd=cmd, win_id=self.win_id, + parent=self._completion) self._completion.selection_changed.connect( completer_obj.on_selection_changed) objreg.register('completion', self._completion, scope='window', diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index 0e8aabb93..012122937 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -65,7 +65,8 @@ def completer_obj(qtbot, status_command_stub, config_stub, monkeypatch, stubs, """Create the completer used for testing.""" monkeypatch.setattr(completer, 'QTimer', stubs.InstaTimer) config_stub.val.completion.show = 'auto' - return completer.Completer(status_command_stub, 0, completion_widget_stub) + return completer.Completer(cmd=status_command_stub, win_id=0, + parent=completion_widget_stub) @pytest.fixture(autouse=True) From 6b65d96fe101517592820f6e9b0efa64d575a294 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 06:32:54 +0100 Subject: [PATCH 236/415] Reformat comment --- scripts/link_pyqt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/link_pyqt.py b/scripts/link_pyqt.py index 82f8cbac5..ad7d45383 100644 --- a/scripts/link_pyqt.py +++ b/scripts/link_pyqt.py @@ -205,7 +205,7 @@ def main(): args = parser.parse_args() if args.tox: - #workaround for the lack of negative factors in tox.ini + # Workaround for the lack of negative factors in tox.ini if 'LINK_PYQT_SKIP' in os.environ: print('LINK_PYQT_SKIP set, exiting...') sys.exit(0) From 2c2d7fe7349cfef6272e3db9d1a9a52878bde88b Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 06:36:42 +0100 Subject: [PATCH 237/415] Copy-paste pylint commands for second environment Otherwise, tox 2.3.1 (shipped with various distributions) fails with: tox.ConfigError: ConfigError: substitution key 'posargs' not found --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2ada5ab58..a45e04c79 100644 --- a/tox.ini +++ b/tox.ini @@ -133,7 +133,8 @@ deps = -r{toxinidir}/misc/requirements/requirements-pylint.txt commands = {envpython} scripts/link_pyqt.py --tox {envdir} - {[testenv:pylint]commands} + {envpython} -m pylint scripts qutebrowser --output-format=colorized --reports=no {posargs} + {envpython} scripts/dev/run_pylint_on_tests.py {toxinidir} --output-format=colorized --reports=no {posargs} [testenv:pylint-master] basepython = python3 From 7ef64c0f87adecb51017327832e17ef64ea1049e Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 06:45:47 +0100 Subject: [PATCH 238/415] Read $PYTHON in every tox.ini environment See #2341 --- doc/install.asciidoc | 4 ++-- tox.ini | 39 ++++++++++----------------------------- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/doc/install.asciidoc b/doc/install.asciidoc index 1dba5fa57..15718ec41 100644 --- a/doc/install.asciidoc +++ b/doc/install.asciidoc @@ -379,8 +379,8 @@ local Qt install instead of installing PyQt in the virtualenv. However, unless you have a new QtWebKit or QtWebEngine available, qutebrowser will not work. It also typically means you'll be using an older release of QtWebEngine. -On Windows, run `tox -e 'mkvenv-win' instead, however make sure that ONLY -Python3 is in your PATH before running tox. +On Windows, run `set PYTHON=C:\path\to\python.exe` (CMD) or ``$Env:PYTHON = +"..."` (Powershell) first. Creating a wrapper script ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tox.ini b/tox.ini index a45e04c79..e59e8b660 100644 --- a/tox.ini +++ b/tox.ini @@ -35,16 +35,7 @@ commands = # other envs [testenv:mkvenv] -basepython = python3 -commands = {envpython} scripts/link_pyqt.py --tox {envdir} -envdir = {toxinidir}/.venv -usedevelop = true -deps = - -r{toxinidir}/requirements.txt - -# This is used for Windows, since binary name is different -[testenv:mkvenv-win] -basepython = python.exe +basepython = {env:PYTHON:python3} commands = {envpython} scripts/link_pyqt.py --tox {envdir} envdir = {toxinidir}/.venv usedevelop = true @@ -61,7 +52,7 @@ deps = {[testenv:mkvenv]deps} # Virtualenv with PyQt5 from PyPI [testenv:mkvenv-pypi] -basepython = python3 +basepython = {env:PYTHON:python3} envdir = {toxinidir}/.venv commands = {envpython} -c "" usedevelop = true @@ -69,19 +60,9 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-pyqt.txt -# This is used for Windows, since binary name is different -[testenv:mkvenv-win-pypi] -basepython = python.exe -commands = {envpython} -c "" -envdir = {toxinidir}/.venv -usedevelop = true -deps = - -r{toxinidir}/requirements.txt - -r{toxinidir}/misc/requirements/requirements-pyqt.txt - [testenv:misc] ignore_errors = true -basepython = python3 +basepython = {env:PYTHON:python3} # For global .gitignore files passenv = HOME deps = @@ -91,7 +72,7 @@ commands = {envpython} scripts/dev/misc_checks.py spelling [testenv:vulture] -basepython = python3 +basepython = {env:PYTHON:python3} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-vulture.txt @@ -101,7 +82,7 @@ commands = {envpython} scripts/dev/run_vulture.py [testenv:vulture-pyqtlink] -basepython = python3 +basepython = {env:PYTHON:python3} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-vulture.txt @@ -137,7 +118,7 @@ commands = {envpython} scripts/dev/run_pylint_on_tests.py {toxinidir} --output-format=colorized --reports=no {posargs} [testenv:pylint-master] -basepython = python3 +basepython = {env:PYTHON:python3} passenv = {[testenv:pylint]passenv} deps = -r{toxinidir}/requirements.txt @@ -149,7 +130,7 @@ commands = {envpython} scripts/dev/run_pylint_on_tests.py --output-format=colorized --reports=no {posargs} [testenv:flake8] -basepython = python3 +basepython = {env:PYTHON:python3} passenv = deps = -r{toxinidir}/requirements.txt @@ -158,7 +139,7 @@ commands = {envpython} -m flake8 {posargs:qutebrowser tests scripts} [testenv:pyroma] -basepython = python3 +basepython = {env:PYTHON:python3} passenv = deps = -r{toxinidir}/misc/requirements/requirements-pyroma.txt @@ -166,7 +147,7 @@ commands = {envdir}/bin/pyroma . [testenv:check-manifest] -basepython = python3 +basepython = {env:PYTHON:python3} passenv = deps = -r{toxinidir}/misc/requirements/requirements-check-manifest.txt @@ -174,7 +155,7 @@ commands = {envdir}/bin/check-manifest --ignore 'qutebrowser/git-commit-id,qutebrowser/html/doc,qutebrowser/html/doc/*,*/__pycache__' [testenv:docs] -basepython = python3 +basepython = {env:PYTHON:python3} whitelist_externals = git passenv = TRAVIS TRAVIS_PULL_REQUEST deps = From a3f57b9a9b32ec64acc1307e410fd5f274e998fb Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:14 +0100 Subject: [PATCH 239/415] Update flake8-builtins from 1.0 to 1.0.post0 --- misc/requirements/requirements-flake8.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index a031778ba..37ceb7f31 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -3,7 +3,7 @@ attrs==17.3.0 flake8==3.5.0 flake8-bugbear==17.4.0 -flake8-builtins==1.0 +flake8-builtins==1.0.post0 flake8-comprehensions==1.4.1 flake8-copyright==0.2.0 flake8-debugger==3.0.0 From 905748f2d0fef6dd503b0eba1bd3b3fa598a3fc6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:16 +0100 Subject: [PATCH 240/415] Update setuptools from 38.2.1 to 38.2.3 --- misc/requirements/requirements-pip.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pip.txt b/misc/requirements/requirements-pip.txt index 4f4b0986c..8ca5a867e 100644 --- a/misc/requirements/requirements-pip.txt +++ b/misc/requirements/requirements-pip.txt @@ -3,6 +3,6 @@ appdirs==1.4.3 packaging==16.8 pyparsing==2.2.0 -setuptools==38.2.1 +setuptools==38.2.3 six==1.11.0 wheel==0.30.0 From a3612a624a651147af6ac9cb98dc45498589146e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:17 +0100 Subject: [PATCH 241/415] Update altgraph from 0.14 to 0.15 --- misc/requirements/requirements-pyinstaller.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pyinstaller.txt b/misc/requirements/requirements-pyinstaller.txt index e542e4243..b2803311f 100644 --- a/misc/requirements/requirements-pyinstaller.txt +++ b/misc/requirements/requirements-pyinstaller.txt @@ -1,6 +1,6 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -altgraph==0.14 +altgraph==0.15 future==0.16.0 macholib==1.8 pefile==2017.11.5 From 96b6f7c4434c3070e7d4ffc36138d37c502e37a0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:19 +0100 Subject: [PATCH 242/415] Update macholib from 1.8 to 1.9 --- misc/requirements/requirements-pyinstaller.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pyinstaller.txt b/misc/requirements/requirements-pyinstaller.txt index b2803311f..f65e8c62a 100644 --- a/misc/requirements/requirements-pyinstaller.txt +++ b/misc/requirements/requirements-pyinstaller.txt @@ -2,6 +2,6 @@ altgraph==0.15 future==0.16.0 -macholib==1.8 +macholib==1.9 pefile==2017.11.5 -e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller From f7ccb8061b7acb00b68a19d625129c34c3c0bee1 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:20 +0100 Subject: [PATCH 243/415] Update pyroma from 2.2 to 2.3 --- misc/requirements/requirements-pyroma.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pyroma.txt b/misc/requirements/requirements-pyroma.txt index d6ed0c190..241273169 100644 --- a/misc/requirements/requirements-pyroma.txt +++ b/misc/requirements/requirements-pyroma.txt @@ -1,4 +1,4 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py docutils==0.14 -pyroma==2.2 +pyroma==2.3 From a0caa2b7b1161126b0f81e83d7350714ce1a3edd Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:22 +0100 Subject: [PATCH 244/415] Update hypothesis from 3.38.5 to 3.40.1 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 358dd72a6..a359d0662 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -11,7 +11,7 @@ fields==5.0.0 Flask==0.12.2 glob2==0.6 hunter==2.0.2 -hypothesis==3.38.5 +hypothesis==3.40.1 itsdangerous==0.24 # Jinja2==2.9.6 Mako==1.0.7 From 71095da975e37c91f44170630b213161bd076930 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:23 +0100 Subject: [PATCH 245/415] Update pytest from 3.2.5 to 3.3.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index a359d0662..cd00dd3e9 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -20,7 +20,7 @@ parse==1.8.2 parse-type==0.4.2 py==1.5.2 py-cpuinfo==3.3.0 -pytest==3.2.5 +pytest==3.3.0 pytest-bdd==2.19.0 pytest-benchmark==3.1.1 pytest-catchlog==1.2.2 From 956c257d196a251cb3f303908dfbebf3affda465 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:25 +0100 Subject: [PATCH 246/415] Update pytest-travis-fold from 1.2.0 to 1.3.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index cd00dd3e9..37fd891db 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -31,7 +31,7 @@ pytest-mock==1.6.3 pytest-qt==2.3.0 pytest-repeat==0.4.1 pytest-rerunfailures==3.1 -pytest-travis-fold==1.2.0 +pytest-travis-fold==1.3.0 pytest-xvfb==1.0.0 PyVirtualDisplay==0.2.1 six==1.11.0 From 2cdc32ca5840b4f4804c97ef1ff6c85053f10714 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 16:55:57 +0100 Subject: [PATCH 247/415] Remove pytest-catchlog --- misc/requirements/requirements-tests-git.txt | 6 ------ misc/requirements/requirements-tests.txt | 1 - misc/requirements/requirements-tests.txt-raw | 1 - tests/helpers/logfail.py | 17 ++++----------- tests/helpers/test_logfail.py | 22 -------------------- tests/unit/utils/test_log.py | 6 +++--- 6 files changed, 7 insertions(+), 46 deletions(-) diff --git a/misc/requirements/requirements-tests-git.txt b/misc/requirements/requirements-tests-git.txt index 6b31140bb..6681dd15e 100644 --- a/misc/requirements/requirements-tests-git.txt +++ b/misc/requirements/requirements-tests-git.txt @@ -12,12 +12,6 @@ git+https://github.com/jenisys/parse_type.git hg+https://bitbucket.org/pytest-dev/py git+https://github.com/pytest-dev/pytest.git@features git+https://github.com/pytest-dev/pytest-bdd.git - -# This is broken at the moment because logfail tries to access -# LogCaptureHandler -# git+https://github.com/eisensheng/pytest-catchlog.git -pytest-catchlog==1.2.2 - git+https://github.com/pytest-dev/pytest-cov.git git+https://github.com/pytest-dev/pytest-faulthandler.git git+https://github.com/pytest-dev/pytest-instafail.git diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 37fd891db..58644a500 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -23,7 +23,6 @@ py-cpuinfo==3.3.0 pytest==3.3.0 pytest-bdd==2.19.0 pytest-benchmark==3.1.1 -pytest-catchlog==1.2.2 pytest-cov==2.5.1 pytest-faulthandler==1.3.1 pytest-instafail==0.3.0 diff --git a/misc/requirements/requirements-tests.txt-raw b/misc/requirements/requirements-tests.txt-raw index bc44bc8e1..121689980 100644 --- a/misc/requirements/requirements-tests.txt-raw +++ b/misc/requirements/requirements-tests.txt-raw @@ -7,7 +7,6 @@ hypothesis pytest pytest-bdd pytest-benchmark -pytest-catchlog pytest-cov pytest-faulthandler pytest-instafail diff --git a/tests/helpers/logfail.py b/tests/helpers/logfail.py index 3d8e3afb8..ba7ed24b8 100644 --- a/tests/helpers/logfail.py +++ b/tests/helpers/logfail.py @@ -22,16 +22,7 @@ import logging import pytest - -try: - import pytest_catchlog as catchlog_mod -except ImportError: - # When using pytest for pyflakes/pep8/..., the plugin won't be available - # but conftest.py will still be loaded. - # - # However, LogFailHandler.emit will never be used in that case, so we just - # ignore the ImportError. - pass +import _pytest.logging class LogFailHandler(logging.Handler): @@ -50,8 +41,8 @@ class LogFailHandler(logging.Handler): return for h in root_logger.handlers: - if isinstance(h, catchlog_mod.LogCaptureHandler): - catchlog_handler = h + if isinstance(h, _pytest.logging.LogCaptureHandler): + capture_handler = h break else: # The LogCaptureHandler is not available anymore during fixture @@ -59,7 +50,7 @@ class LogFailHandler(logging.Handler): return if (logger.level == record.levelno or - catchlog_handler.level == record.levelno): + capture_handler.level == record.levelno): # caplog.at_level(...) was used with the level of this message, # i.e. it was expected. return diff --git a/tests/helpers/test_logfail.py b/tests/helpers/test_logfail.py index b95dec1d6..48aaaa201 100644 --- a/tests/helpers/test_logfail.py +++ b/tests/helpers/test_logfail.py @@ -23,7 +23,6 @@ import logging import pytest -import pytest_catchlog def test_log_debug(): @@ -64,24 +63,3 @@ def test_log_expected_wrong_logger(caplog): with pytest.raises(pytest.fail.Exception): with caplog.at_level(logging.ERROR, logger): logging.error('foo') - - -@pytest.fixture -def skipping_fixture(): - pytest.skip("Skipping to test caplog workaround.") - - -def test_caplog_bug_workaround_1(caplog, skipping_fixture): - pass - - -def test_caplog_bug_workaround_2(): - """Make sure caplog_bug_workaround works correctly after a skipped test. - - There should be only one capturelog handler. - """ - caplog_handler = None - for h in logging.getLogger().handlers: - if isinstance(h, pytest_catchlog.LogCaptureHandler): - assert caplog_handler is None - caplog_handler = h diff --git a/tests/unit/utils/test_log.py b/tests/unit/utils/test_log.py index 30dd5d634..2ab51002d 100644 --- a/tests/unit/utils/test_log.py +++ b/tests/unit/utils/test_log.py @@ -27,7 +27,7 @@ import warnings import attr import pytest -import pytest_catchlog +import _pytest.logging from qutebrowser.utils import log from qutebrowser.misc import utilcmds @@ -66,11 +66,11 @@ def restore_loggers(): while root_logger.handlers: h = root_logger.handlers[0] root_logger.removeHandler(h) - if not isinstance(h, pytest_catchlog.LogCaptureHandler): + if not isinstance(h, _pytest.logging.LogCaptureHandler): h.close() root_logger.setLevel(original_logging_level) for h in root_handlers: - if not isinstance(h, pytest_catchlog.LogCaptureHandler): + if not isinstance(h, _pytest.logging.LogCaptureHandler): # https://github.com/qutebrowser/qutebrowser/issues/856 root_logger.addHandler(h) logging._acquireLock() From 6973a703c54783b774079bab40f4d75e64262e17 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 16:56:39 +0100 Subject: [PATCH 248/415] Add pluggy to requirements --- misc/requirements/requirements-tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 58644a500..119891725 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -18,6 +18,7 @@ Mako==1.0.7 # MarkupSafe==1.0 parse==1.8.2 parse-type==0.4.2 +pluggy==0.6.0 py==1.5.2 py-cpuinfo==3.3.0 pytest==3.3.0 From b6466b74108d5087f3b5659d75dd4ab0e0e1ac93 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 4 Dec 2017 13:08:56 -0300 Subject: [PATCH 249/415] revision 2 --- scripts/hist_importer.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 79c723c58..36f151e6f 100755 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -2,7 +2,7 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: # Copyright 2017 Florian Bruhin (The Compiler) -# Copyright 2014-2017 Josefson Souza +# Copyright 2017 Josefson Souza # This file is part of qutebrowser. # @@ -32,17 +32,17 @@ import os def parse(): """Parse command line arguments.""" description = ("This program is meant to extract browser history from your" - "previous browser and import them into qutebrowser.") - epilog = ("Databases:\n\tQutebrowser: Is named 'history.sqlite' and can be" + " previous browser and import them into qutebrowser.") + epilog = ("Databases:\n\n\tqutebrowser: Is named 'history.sqlite' and can be" " found at your --basedir. In order to find where your basedir" " is you can run ':open qute:version' inside qutebrowser." - "\n\tFirerox: Is named 'places.sqlite', and can be found at your" - "system\"s profile folder. Check this link for where it is locat" - "ed: http://kb.mozillazine.org/Profile_folder" - "\n\tChrome: Is named 'History', and can be found at the respec" - "tive User Data Directory. Check this link for where it is locat" - "ed: https://chromium.googlesource.com/chromium/src/+/master/" - "docs/user_data_dir.md\n\n" + "\n\n\tFirefox: Is named 'places.sqlite', and can be found at your" + " system's profile folder. Check this link for where it is " + "located: http://kb.mozillazine.org/Profile_folder" + "\n\n\tChrome: Is named 'History', and can be found at the " + "respective User Data Directory. Check this link for where it is" + "located: https://chromium.googlesource.com/chromium/src/+/" + "master/docs/user_data_dir.md\n\n" "Example: hist_importer.py -b firefox -s /Firefox/Profile/" "places.sqlite -d /qutebrowser/data/history.sqlite") parser = argparse.ArgumentParser( @@ -55,7 +55,7 @@ def parse(): type=str, help='Source: Full path to the sqlite data' 'base file from the source browser.') parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, - help='Destination: The full path to the qutebrowser ' + help='\nDestination: Full path to the qutebrowser ' 'sqlite database') return parser.parse_args() @@ -66,10 +66,9 @@ def open_db(data_base): conn = sqlite3.connect(data_base) return conn else: - raise sys.exit('\nDataBaseNotFound: There was some error trying to to' + raise sys.exit('DataBaseNotFound: There was some error trying to to' ' connect with the [{}] database. Verify if the' - ' filepath is correct or is being used.' - .format(data_base)) + ' given file path is correct.'.format(data_base)) def extract(source, query): @@ -91,8 +90,7 @@ def extract(source, query): return history except sqlite3.OperationalError as op_e: print('\nCould not perform queries into the source database: {}' - '\nBrowser version is not supported as it have a different sql' - ' schema.'.format(op_e)) + .format(op_e)) def clean(history): From 4467f51e42275b1662b6aab7d7de073a5b0e6b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Mon, 4 Dec 2017 18:15:02 +0100 Subject: [PATCH 250/415] Use 'language: generic' for shellcheck, fix typo, correct indentation --- .travis.yml | 1 + misc/userscripts/rss | 4 ++-- scripts/dev/ci/travis_run.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37387b620..0ea8218a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,6 +53,7 @@ matrix: python: null node_js: "lts/*" - os: linux + language: generic env: TESTENV=shellcheck services: docker fast_finish: true diff --git a/misc/userscripts/rss b/misc/userscripts/rss index 6680a259e..f8feebee7 100755 --- a/misc/userscripts/rss +++ b/misc/userscripts/rss @@ -57,7 +57,7 @@ notice () { # Update a database of a feed and open new URLs read_items () { - cd read_urls || return + cd read_urls || return 1 feed_file="$(echo "$1" | tr -d /)" feed_temp_file="$(mktemp "$feed_file.tmp.XXXXXXXXXX")" feed_new_items="$(mktemp "$feed_file.new.XXXXXXXXXX")" @@ -85,7 +85,7 @@ if [ ! -d "$config_dir/read_urls" ]; then mkdir -p "$config_dir/read_urls" fi -cd "$config_dir" || exit +cd "$config_dir" || exit 1 if [ $# != 0 ]; then for arg in "$@"; do diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index 74b8ae63f..c0f59dc1a 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -22,7 +22,7 @@ elif [[ $TESTENV == shellcheck ]]; then IFS=" " read -r -a scripts <<< "$dev_scripts $userscripts" docker run \ -v "$PWD:/outside" \ - -w /outside \ + -w /outside \ koalaman/shellcheck:latest "${scripts[@]}" else args=() From 86c37538d71e2937386ae89089191018231c7998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Mon, 4 Dec 2017 18:29:55 +0100 Subject: [PATCH 251/415] Simply search for shell scripts to search Use 2 simpler find commands and redirect the output to a temporary file. --- scripts/dev/ci/travis_run.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index c0f59dc1a..b7d44968e 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -15,11 +15,11 @@ elif [[ $TESTENV == eslint ]]; then cd qutebrowser/javascript || exit 1 eslint --color --report-unused-disable-directives . elif [[ $TESTENV == shellcheck ]]; then - dev_scripts=$( find scripts/dev/ -name '*.sh' -print0 | xargs -0 ) - # false positive: we are using 'find -exec +' - # shellcheck disable=SC2038 - userscripts=$( find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + | xargs ) - IFS=" " read -r -a scripts <<< "$dev_scripts $userscripts" + SCRIPTS=$( mktemp ) + find scripts/dev/ -name '*.sh' >"$SCRIPTS" + find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + >>"$SCRIPTS" + mapfile -t scripts <"$SCRIPTS" + rm -f "$SCRIPTS" docker run \ -v "$PWD:/outside" \ -w /outside \ From 3a04de62ae4a5dde8aabd4edcc7a7e584c113aae Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 19:01:21 +0100 Subject: [PATCH 252/415] Recompile requirements --- misc/requirements/requirements-tests.txt | 2 +- misc/requirements/requirements-tox.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 119891725..46905f497 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -13,7 +13,7 @@ glob2==0.6 hunter==2.0.2 hypothesis==3.40.1 itsdangerous==0.24 -# Jinja2==2.9.6 +# Jinja2==2.10 Mako==1.0.7 # MarkupSafe==1.0 parse==1.8.2 diff --git a/misc/requirements/requirements-tox.txt b/misc/requirements/requirements-tox.txt index 1308c8afd..d2b3a719b 100644 --- a/misc/requirements/requirements-tox.txt +++ b/misc/requirements/requirements-tox.txt @@ -2,5 +2,6 @@ pluggy==0.6.0 py==1.5.2 +six==1.11.0 tox==2.9.1 virtualenv==15.1.0 From 28caddf3c1700f77c78173b3677abd71c8d9582a Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Mon, 4 Dec 2017 19:02:09 +0100 Subject: [PATCH 253/415] delay added, text changed --- qutebrowser/html/back.html | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index b6190feda..cfaae589b 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -21,6 +21,15 @@ function go_forward() { history.forward(); } +function prepare_restore() { + if (!document.hidden) { + go_back(); + break; + } + + document.addEventListener("visibilitychange", go_back); +} + // there are three states // default: register focus listener, // on focus: go back and switch to the state forward @@ -38,17 +47,14 @@ switch (history.state) { go_forward(); break; default: - if (!document.hidden) { - go_back(); - break; - } - - document.addEventListener("visibilitychange", go_back); + setTimeout(prepare_restore, 1000); break; } {% endblock %} {% block content %} -

If you see this site something went wrong, or you reached the end of the history, or you disabled javascript.

+

Loading suspended page...
+
+If nothing happens, something went wrong or you disabled JavaScript.

{% endblock %} From 02104a318e9c8dfea690447a544f84c2dd19197b Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Mon, 4 Dec 2017 19:03:12 +0100 Subject: [PATCH 254/415] delay added, text changed --- qutebrowser/html/back.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index cfaae589b..46945ab65 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -24,7 +24,7 @@ function go_forward() { function prepare_restore() { if (!document.hidden) { go_back(); - break; + return; } document.addEventListener("visibilitychange", go_back); From 9675ea93eebb51bcb80480a248ad209fc957dc29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Mon, 4 Dec 2017 20:31:28 +0100 Subject: [PATCH 255/415] Do not call pip in travis_install.sh when TESTENV=shellcheck --- scripts/dev/ci/travis_install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index 18ac44048..04b118b9b 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -96,6 +96,8 @@ case $TESTENV in eslint) npm_install eslint ;; + shellcheck) + ;; *) pip_install pip pip_install -r misc/requirements/requirements-tox.txt From b554e1f7639d340fa704396b61cd7346914a28f6 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 22:07:23 +0100 Subject: [PATCH 256/415] tests: Add after= argument to wait_for --- tests/end2end/fixtures/testprocess.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/end2end/fixtures/testprocess.py b/tests/end2end/fixtures/testprocess.py index a4b136193..c210f6fe7 100644 --- a/tests/end2end/fixtures/testprocess.py +++ b/tests/end2end/fixtures/testprocess.py @@ -333,7 +333,7 @@ class Process(QObject): else: return value == expected - def _wait_for_existing(self, override_waited_for, **kwargs): + def _wait_for_existing(self, override_waited_for, after, **kwargs): """Check if there are any line in the history for wait_for. Return: either the found line or None. @@ -345,7 +345,15 @@ class Process(QObject): value = getattr(line, key) matches.append(self._match_data(value, expected)) - if all(matches) and (not line.waited_for or override_waited_for): + if after is None: + too_early = False + else: + too_early = ((line.timestamp, line.msecs) < + (after.timestamp, after.msecs)) + + if (all(matches) and + (not line.waited_for or override_waited_for) and + not too_early): # If we waited for this line, chances are we don't mean the # same thing the next time we use wait_for and it matches # this line again. @@ -422,7 +430,7 @@ class Process(QObject): pass def wait_for(self, timeout=None, *, override_waited_for=False, - do_skip=False, divisor=1, **kwargs): + do_skip=False, divisor=1, after=None, **kwargs): """Wait until a given value is found in the data. Keyword arguments to this function get interpreted as attributes of the @@ -435,6 +443,7 @@ class Process(QObject): again. do_skip: If set, call pytest.skip on a timeout. divisor: A factor to decrease the timeout by. + after: If it's an existing line, ensure it's after the given one. Return: The matched line. @@ -456,7 +465,8 @@ class Process(QObject): for key in kwargs: assert key in self.KEYS - existing = self._wait_for_existing(override_waited_for, **kwargs) + existing = self._wait_for_existing(override_waited_for, after, + **kwargs) if existing is not None: return existing else: From a8f4444c24f785dfc24c107f67d5f8a2208d95d1 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 22:07:49 +0100 Subject: [PATCH 257/415] tests: Show more of the message --- tests/end2end/fixtures/testprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/fixtures/testprocess.py b/tests/end2end/fixtures/testprocess.py index c210f6fe7..bc987043f 100644 --- a/tests/end2end/fixtures/testprocess.py +++ b/tests/end2end/fixtures/testprocess.py @@ -371,7 +371,7 @@ class Process(QObject): __tracebackhide__ = lambda e: e.errisinstance(WaitForTimeout) message = kwargs.get('message', None) if message is not None: - elided = quteutils.elide(repr(message), 50) + elided = quteutils.elide(repr(message), 100) self._log("\n----> Waiting for {} in the log".format(elided)) spy = QSignalSpy(self.new_data) From 62228752aae48367b9393288f0b7cf90cace1401 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 5 Dec 2017 07:05:07 +0100 Subject: [PATCH 258/415] Fix most end2end tests with Qt 5.10 For some reason, if we don't wait for about:blank to be fully loaded with Qt 5.10, we get the next LoadStatus.finished notification with about:blank as URL. This is most likely caused by the changes in https://codereview.qt-project.org/#/c/202924/ See #3003 --- tests/end2end/fixtures/quteprocess.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index 27c347ca4..454ae80a5 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -36,7 +36,7 @@ import pytest from PyQt5.QtCore import pyqtSignal, QUrl, qVersion from qutebrowser.misc import ipc -from qutebrowser.utils import log, utils, javascript +from qutebrowser.utils import log, utils, javascript, qtutils from helpers import utils as testutils from end2end.fixtures import testprocess @@ -687,9 +687,12 @@ class QuteProc(testprocess.Process): raise ValueError("Invalid URL {}: {}".format(url, qurl.errorString())) - if qurl == QUrl('about:blank'): + if (qurl == QUrl('about:blank') and + not qtutils.version_check('5.10', compiled=False)): # For some reason, we don't get a LoadStatus.success for # about:blank sometimes. + # However, if we do this for Qt 5.10, we get general testsuite + # instability as site loads get reported with about:blank... pattern = "Changing title for idx * to 'about:blank'" else: # We really need the same representation that the webview uses in From 0ce9a355aed3a7df1932c0b7bda34aeb1e5d3ef4 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 5 Dec 2017 07:07:13 +0100 Subject: [PATCH 259/415] Fix download test with Qt 5.10 Not sure why this is needed (no prompt is shown otherwise), but it works like this. This probably is related to https://bugreports.qt.io/browse/QTBUG-63388 See #3003 --- tests/end2end/features/downloads.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/features/downloads.feature b/tests/end2end/features/downloads.feature index 69f47603b..833f63cb4 100644 --- a/tests/end2end/features/downloads.feature +++ b/tests/end2end/features/downloads.feature @@ -536,7 +536,7 @@ Feature: Downloading things from a website. And I open data/downloads/download.bin without waiting And I wait for the download prompt for "*" And I run :prompt-accept (tmpdir)(dirsep)downloads - And I open data/downloads/download.bin without waiting + And I open data/downloads/download2.bin without waiting And I wait for the download prompt for "*" And I directly open the download And I open data/downloads/download.bin without waiting From 29c2e7b45f75cf6b1b3cdd4fb6cee341f20c7601 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 5 Dec 2017 07:09:43 +0100 Subject: [PATCH 260/415] Skip :follow-selected tests on Qt 5.10 See #3003, #2635 --- tests/end2end/features/search.feature | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/end2end/features/search.feature b/tests/end2end/features/search.feature index 3778f963d..56fcca207 100644 --- a/tests/end2end/features/search.feature +++ b/tests/end2end/features/search.feature @@ -225,11 +225,15 @@ Feature: Searching on a page Then the following tabs should be open: - data/search.html (active) + # Following a link selected via JS doesn't work in Qt 5.10 anymore. + @qt!=5.10 Scenario: Follow a manually selected link When I run :jseval --file (testdata)/search_select.js And I run :follow-selected Then data/hello.txt should be loaded + # Following a link selected via JS doesn't work in Qt 5.10 anymore. + @qt!=5.10 Scenario: Follow a manually selected link in a new tab When I run :window-only And I run :jseval --file (testdata)/search_select.js From 8a3437c6a4edabf2212a8e37e9e8490be18fa1b0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 5 Dec 2017 09:36:14 +0100 Subject: [PATCH 261/415] Adjust Debian install instructions --- doc/install.asciidoc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/install.asciidoc b/doc/install.asciidoc index 15718ec41..312e53176 100644 --- a/doc/install.asciidoc +++ b/doc/install.asciidoc @@ -35,7 +35,7 @@ Ubuntu 16.04 doesn't come with an up-to-date engine (a new enough QtWebKit, or QtWebEngine). However, it comes with Python 3.5, so you can <>. -Debian Stretch / Ubuntu 17.04 and newer +Debian Stretch / Ubuntu 17.04 and 17.10 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Those versions come with QtWebEngine in the repositories. This makes it possible @@ -54,7 +54,18 @@ Install the packages: # apt install ./qutebrowser_*_all.deb ---- -Some additional hints: +Debian Testing / Ubuntu 18.04 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Debian Testing, qutebrowser is in the official repositories, and you can +install it with apt: + +---- +# apt install qutebrowser +---- + +Additional hints +~~~~~~~~~~~~~~~~ - Alternatively, you can <> to get a newer QtWebEngine version. From 636f9edff62fb121bb52a1c25ec5903d24a18074 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 5 Dec 2017 07:31:55 -0500 Subject: [PATCH 262/415] History completion by both URL and title. Resolves #1649. --- qutebrowser/completion/models/histcategory.py | 2 +- tests/unit/completion/test_histcategory.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index b993b40de..fe89dc79b 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -47,7 +47,7 @@ class HistoryCategory(QSqlQueryModel): "FROM CompletionHistory", # the incoming pattern will have literal % and _ escaped with '\' # we need to tell sql to treat '\' as an escape character - "WHERE (url LIKE :pat escape '\\' or title LIKE :pat escape '\\')", + "WHERE ((url || title) LIKE :pat escape '\\')", self._atime_expr(), "ORDER BY last_atime DESC", ]), forward_only=False) diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index 834b3a5a3..b87eb6ac2 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -78,6 +78,10 @@ def hist(init_sql, config_stub): ("can't", [("can't touch this", ''), ('a', '')], [("can't touch this", '')]), + + ("ample itle", + [('example.com', 'title'), ('example.com', 'nope')], + [('example.com', 'title')]), ]) def test_set_pattern(pattern, before, after, model_validator, hist): """Validate the filtering and sorting results of set_pattern.""" From 00a09354c3231d78700187a33ac49d2bbf1aca30 Mon Sep 17 00:00:00 2001 From: Justin Partain Date: Tue, 5 Dec 2017 08:28:10 -0500 Subject: [PATCH 263/415] Track number of active searches in tab, ignore all but most recent search callbacks --- qutebrowser/browser/webengine/webenginetab.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 7339cd422..b40d8b871 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -126,13 +126,19 @@ class WebEngineSearch(browsertab.AbstractSearch): def __init__(self, parent=None): super().__init__(parent) self._flags = QWebEnginePage.FindFlags(0) + self.num_of_searches = 0 def _find(self, text, flags, callback, caller): """Call findText on the widget.""" self.search_displayed = True + self.num_of_searches += 1 def wrapped_callback(found): """Wrap the callback to do debug logging.""" + self.num_of_searches -= 1 + if self.num_of_searches > 0: + return + found_text = 'found' if found else "didn't find" if flags: flag_text = 'with flags {}'.format(debug.qflags_key( From 7f81f0c0ab79f9082d807f23561060f12cd9d05f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 06:51:15 +0100 Subject: [PATCH 264/415] Always open session tabs in foreground This helps with issues with lazy sessions as document.hidden was set incorrectly. See #3345, #3366 --- qutebrowser/misc/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 237e46189..bfd73ef32 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -422,7 +422,7 @@ class SessionManager(QObject): window=window.win_id) tab_to_focus = None for i, tab in enumerate(win['tabs']): - new_tab = tabbed_browser.tabopen() + new_tab = tabbed_browser.tabopen(background=False) self._load_tab(new_tab, tab) if tab.get('active', False): tab_to_focus = i From 7a6d568c8cd980e3b57c678070dfdea387245089 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 06:53:27 +0100 Subject: [PATCH 265/415] Remove blank line --- qutebrowser/html/back.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index 46945ab65..894427800 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -26,7 +26,7 @@ function prepare_restore() { go_back(); return; } - + document.addEventListener("visibilitychange", go_back); } From 58212a7b15864b2a1b3b6afa73972908032f0e02 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 06:56:12 +0100 Subject: [PATCH 266/415] Update docs --- doc/changelog.asciidoc | 2 ++ doc/help/settings.asciidoc | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 34ee4ff7c..19714700a 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -51,6 +51,8 @@ Added - New `:edit-command` command to edit the commandline in an editor. - New `tabs.persist_mode_on_change` setting to keep the current mode when switching tabs. +- New `session.lazy_restore` setting which allows to not load pages immediately + when restoring a session. Changed ~~~~~~~ diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 2da7a5008..967f6c4c6 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -223,6 +223,7 @@ |<>|Show a scrollbar. |<>|Enable smooth scrolling for web pages. |<>|Name of the session to save by default. +|<>|Load a restored tab as soon as it takes focus. |<>|Languages to use for spell checking. |<>|Hide the statusbar unless a message is shown. |<>|Padding (in pixels) for the statusbar. @@ -2565,6 +2566,14 @@ Type: <> Default: empty +[[session.lazy_restore]] +=== session.lazy_restore +Load a restored tab as soon as it takes focus. + +Type: <> + +Default: +pass:[false]+ + [[spellcheck.languages]] === spellcheck.languages Languages to use for spell checking. From 9c042e4313a7f1229e2bb8169f6151681937e8f9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 06:58:22 +0100 Subject: [PATCH 267/415] Update changelog --- doc/changelog.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 19714700a..f734d96bc 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -91,6 +91,8 @@ Changed data dir, e.g. `~/.local/share/qutebrowser/js`. - The current/default bindings are now shown in the :bind completion. - Empty categories are now hidden in the `:open` completion. +- Search terms for URLs and titles can now be mixed when filtering the + completion. Fixed ~~~~~ From b326f1242766212fb593ec7c0adc4dc0e77ddc98 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 06:59:08 +0100 Subject: [PATCH 268/415] Mark editor test as flaky See #3367 --- tests/end2end/features/editor.feature | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 21f3df425..3e1be47bc 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -117,6 +117,8 @@ Feature: Opening external editors # Could not get signals working on Windows @posix + # There's no guarantee that the tab gets deleted... + @flaky Scenario: Spawning an editor and closing the tab When I set up a fake editor that waits And I open data/editor.html From 549a3a8f70686b1eb2e3a583c477482c94add605 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 07:41:41 +0100 Subject: [PATCH 269/415] Improve hist_importer messages --- scripts/hist_importer.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 36f151e6f..ba3369f5c 100755 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -66,9 +66,7 @@ def open_db(data_base): conn = sqlite3.connect(data_base) return conn else: - raise sys.exit('DataBaseNotFound: There was some error trying to to' - ' connect with the [{}] database. Verify if the' - ' given file path is correct.'.format(data_base)) + sys.exit('The file {} does not exist.'.format(data_base)) def extract(source, query): @@ -89,8 +87,8 @@ def extract(source, query): conn.close() return history except sqlite3.OperationalError as op_e: - print('\nCould not perform queries into the source database: {}' - .format(op_e)) + sys.exit('Could not perform queries on the source database: ' + '{}'.format(op_e)) def clean(history): From 16e09d18fa1207ccb78d6274965388c91a901f4c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 07:42:07 +0100 Subject: [PATCH 270/415] Update changelog --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index f734d96bc..4acd841b2 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -53,6 +53,7 @@ Added switching tabs. - New `session.lazy_restore` setting which allows to not load pages immediately when restoring a session. +- New `hist_importer.py` script to import history from Firefox/Chromium. Changed ~~~~~~~ From a3ba7b9f60393445cce7e6128e9a82bcfade5f42 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 07:45:52 +0100 Subject: [PATCH 271/415] Reformat hist_importer epilog. --- scripts/hist_importer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index ba3369f5c..f4ad47062 100755 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -33,11 +33,11 @@ def parse(): """Parse command line arguments.""" description = ("This program is meant to extract browser history from your" " previous browser and import them into qutebrowser.") - epilog = ("Databases:\n\n\tqutebrowser: Is named 'history.sqlite' and can be" - " found at your --basedir. In order to find where your basedir" - " is you can run ':open qute:version' inside qutebrowser." - "\n\n\tFirefox: Is named 'places.sqlite', and can be found at your" - " system's profile folder. Check this link for where it is " + epilog = ("Databases:\n\n\tqutebrowser: Is named 'history.sqlite' and can " + "be found at your --basedir. In order to find where your " + "basedir is you can run ':open qute:version' inside qutebrowser." + "\n\n\tFirefox: Is named 'places.sqlite', and can be found at " + "your system's profile folder. Check this link for where it is " "located: http://kb.mozillazine.org/Profile_folder" "\n\n\tChrome: Is named 'History', and can be found at the " "respective User Data Directory. Check this link for where it is" From 1a3f8662e6288cf46d0a9daf61d220718b94d448 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 07:56:59 +0100 Subject: [PATCH 272/415] Improve handling of cancelled search callbacks --- qutebrowser/browser/webengine/webenginetab.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 32bb95ca1..5a656e2b5 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -121,22 +121,33 @@ class WebEnginePrinting(browsertab.AbstractPrinting): class WebEngineSearch(browsertab.AbstractSearch): - """QtWebEngine implementations related to searching on the page.""" + """QtWebEngine implementations related to searching on the page. + + Attributes: + _flags: The QWebEnginePage.FindFlags of the last search. + _pending_searches: How many searches have been started but not called + back yet. + """ def __init__(self, parent=None): super().__init__(parent) self._flags = QWebEnginePage.FindFlags(0) - self.num_of_searches = 0 + self._pending_searches = 0 def _find(self, text, flags, callback, caller): """Call findText on the widget.""" self.search_displayed = True - self.num_of_searches += 1 + self._pending_searches += 1 def wrapped_callback(found): """Wrap the callback to do debug logging.""" - self.num_of_searches -= 1 - if self.num_of_searches > 0: + self._pending_searches -= 1 + if self._pending_searches > 0: + # See https://github.com/qutebrowser/qutebrowser/issues/2442 + # and https://github.com/qt/qtwebengine/blob/5.10/src/core/web_contents_adapter.cpp#L924-L934 + log.webview.debug("Ignoring cancelled search callback with " + "{} pending searches".format( + self._pending_searches)) return found_text = 'found' if found else "didn't find" From 7d7c8412500ae4e486d8d88c6db97e457d8921ce Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 08:01:39 +0100 Subject: [PATCH 273/415] Update changelog --- doc/changelog.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 4acd841b2..2302a65c4 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -112,6 +112,8 @@ Fixed in a URL. - Using e.g. `-s backend webkit` to set the backend now works correctly. - Fixed crash when closing the tab an external editor was opened in. +- When using `:search-next` before a search is finished, no warning about no + results being found is shown anymore. Deprecated ~~~~~~~~~~ From 129f97873a57d4b56b9c399e9da5c9da70dd880c Mon Sep 17 00:00:00 2001 From: Jimmy Date: Mon, 27 Nov 2017 20:07:16 +1300 Subject: [PATCH 274/415] Greasemonkey: add assert to tests scripts_for assumptions. And crash the users browsing session as a result of any accidental and totally, otherwise, non-fatal unforseen errors. --- qutebrowser/browser/webkit/webpage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 89b293869..02aa270d7 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -323,6 +323,8 @@ class BrowserPage(QWebPage): # also indicate a bug. log.greasemonkey.debug("Not running scripts for frame with no " "url: {}".format(frame)) + assert not toload, toload + for script in toload: if frame is self.mainFrame() or script.runs_on_sub_frames: log.webview.debug('Running GM script: {}'.format(script.name)) From ead108eeebadfc6e73b5927901eb155eebad028b Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 6 Dec 2017 20:27:56 +1300 Subject: [PATCH 275/415] fixup! Greasemonkey: Add run-at document-idle. --- qutebrowser/browser/greasemonkey.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 845b83d85..3fd01137f 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -193,8 +193,8 @@ class GreasemonkeyManager(QObject): match = functools.partial(fnmatch.fnmatch, url.toString(QUrl.FullyEncoded)) tester = (lambda script: - any([match(pat) for pat in script.includes]) and - not any([match(pat) for pat in script.excludes])) + any(match(pat) for pat in script.includes) and + not any(match(pat) for pat in script.excludes)) return MatchingScripts( url, [script for script in self._run_start if tester(script)], From 6b3e16b1630acfc8a78f4a8994d9ad2279450386 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 6 Dec 2017 20:34:29 +1300 Subject: [PATCH 276/415] Greasemonkey: mark failing no(sub)frames test as flaky. This test is supposed to ensure that user scripts don't run on iframes when the @noframes directive is set in the greasemonkey metadata. It is failing sometimes on travis but passing on local test runs. Personally I haven't actually ran the whole test suite through, just the javascript tests. It maybe be some stale state that only shows up when you run the whole suite. It may be some timing issue that only shows up on travis because ???. Hopefully this stops the red x from showing up on the PR. --- tests/end2end/features/javascript.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature index aaad84be3..944d2606d 100644 --- a/tests/end2end/features/javascript.feature +++ b/tests/end2end/features/javascript.feature @@ -139,6 +139,7 @@ Feature: Javascript stuff And I open data/hints/iframe.html Then the javascript message "Script is running on /data/hints/html/wrapped.html" should be logged + @flaky Scenario: Have a greasemonkey script running on noframes When I have a greasemonkey file saved for document-end with noframes set And I run :greasemonkey-reload From 0c792d228e4055c7195ee7b314bb522c8294dd84 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 11:12:25 +0100 Subject: [PATCH 277/415] Update docs --- doc/changelog.asciidoc | 2 ++ doc/help/commands.asciidoc | 7 +++++++ qutebrowser/browser/greasemonkey.py | 6 +++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 2302a65c4..16075db27 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -21,6 +21,8 @@ v1.1.0 (unreleased) Added ~~~~~ +- Initial support for Greasemonkey scripts. There are still some rough edges, + but many scripts should already work. - There's now a `misc/Makefile` file in releases, which should help distributions which package qutebrowser, as they can run something like `make -f misc/Makefile DESTDIR="$pkgdir" install` now. diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 0fb99f260..5d026bfca 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -54,6 +54,7 @@ It is possible to run or bind multiple commands by separating them with `;;`. |<>|Follow the selected text. |<>|Go forward in the history of the current tab. |<>|Toggle fullscreen mode. +|<>|Re-read Greasemonkey scripts from disk. |<>|Show help about a command or setting. |<>|Start hinting. |<>|Show browsing history. @@ -491,6 +492,12 @@ Toggle fullscreen mode. ==== optional arguments * +*-l*+, +*--leave*+: Only leave fullscreen if it was entered by the page. +[[greasemonkey-reload]] +=== greasemonkey-reload +Re-read Greasemonkey scripts from disk. + +The scripts are read from a 'greasemonkey' subdirectory in qutebrowser's data directory (see `:version`). + [[help]] === help Syntax: +:help [*--tab*] [*--bg*] [*--window*] ['topic']+ diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 3fd01137f..1a1ec6a24 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -149,7 +149,11 @@ class GreasemonkeyManager(QObject): @cmdutils.register(name='greasemonkey-reload', instance='greasemonkey') def load_scripts(self): - """Re-Read greasemonkey scripts from disk.""" + """Re-read Greasemonkey scripts from disk. + + The scripts are read from a 'greasemonkey' subdirectory in qutebrowser's + data directory (see `:version`). + """ self._run_start = [] self._run_end = [] self._run_idle = [] From dd63508be772679b4a0e684f8c00c1a65e3e9861 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 11:30:01 +0100 Subject: [PATCH 278/415] Add a greasemonkey.init() This also creates the greasemonkey directory if it doesn't exist yet, for discoverability. --- qutebrowser/app.py | 3 +-- qutebrowser/browser/greasemonkey.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index c32a208ac..7029f8df5 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -492,8 +492,7 @@ def _init_modules(args, crash_handler): objreg.register('cache', diskcache) log.init.debug("Initializing Greasemonkey...") - gm_manager = greasemonkey.GreasemonkeyManager() - objreg.register('greasemonkey', gm_manager) + greasemonkey.init() log.init.debug("Misc initialization...") macros.init() diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 1a1ec6a24..669b9d0bc 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -29,7 +29,7 @@ import glob import attr from PyQt5.QtCore import pyqtSignal, QObject, QUrl -from qutebrowser.utils import log, standarddir, jinja +from qutebrowser.utils import log, standarddir, jinja, objreg from qutebrowser.commands import cmdutils @@ -209,3 +209,14 @@ class GreasemonkeyManager(QObject): def all_scripts(self): """Return all scripts found in the configured script directory.""" return self._run_start + self._run_end + self._run_idle + + +def init(): + """Initialize Greasemonkey support.""" + gm_manager = GreasemonkeyManager() + objreg.register('greasemonkey', gm_manager) + + try: + os.mkdir(_scripts_dir()) + except FileExistsError: + pass From 2633dcc0d572da84331362857872a3cd5af27e35 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 11:50:59 +0100 Subject: [PATCH 279/415] Fix lint --- qutebrowser/browser/greasemonkey.py | 24 ++++++------ .../browser/webengine/webenginesettings.py | 13 +++---- qutebrowser/browser/webengine/webview.py | 2 +- qutebrowser/browser/webkit/webpage.py | 4 +- .../javascript/greasemonkey_wrapper.js | 38 +++++++++---------- tests/end2end/features/javascript.feature | 12 +++--- tests/end2end/features/test_javascript_bdd.py | 4 +- tests/unit/javascript/test_greasemonkey.py | 6 +-- 8 files changed, 51 insertions(+), 52 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 669b9d0bc..9a82d6a93 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""Load, parse and make avalaible greasemonkey scripts.""" +"""Load, parse and make available Greasemonkey scripts.""" import re import os @@ -39,6 +39,7 @@ def _scripts_dir(): class GreasemonkeyScript: + """Container class for userscripts, parses metadata blocks.""" def __init__(self, properties, code): @@ -72,15 +73,15 @@ class GreasemonkeyScript: @classmethod def parse(cls, source): - """GreaseMonkeyScript factory. + """GreasemonkeyScript factory. - Takes a userscript source and returns a GreaseMonkeyScript. - Parses the greasemonkey metadata block, if present, to fill out + Takes a userscript source and returns a GreasemonkeyScript. + Parses the Greasemonkey metadata block, if present, to fill out attributes. """ matches = re.split(cls.HEADER_REGEX, source, maxsplit=2) try: - _, props, _code = matches + _head, props, _code = matches except ValueError: props = "" script = cls(re.findall(cls.PROPS_REGEX, props), source) @@ -90,9 +91,9 @@ class GreasemonkeyScript: return script def code(self): - """Return the processed javascript code of this script. + """Return the processed JavaScript code of this script. - Adorns the source code with GM_* methods for greasemonkey + Adorns the source code with GM_* methods for Greasemonkey compatibility and wraps it in an IFFE to hide it within a lexical scope. Note that this means line numbers in your browser's debugger/inspector will not match up to the line @@ -118,6 +119,7 @@ class GreasemonkeyScript: @attr.s class MatchingScripts(object): + """All userscripts registered to run on a particular url.""" url = attr.ib() @@ -128,11 +130,11 @@ class MatchingScripts(object): class GreasemonkeyManager(QObject): - """Manager of userscripts and a greasemonkey compatible environment. + """Manager of userscripts and a Greasemonkey compatible environment. Signals: scripts_reloaded: Emitted when scripts are reloaded from disk. - Any any cached or already-injected scripts should be + Any cached or already-injected scripts should be considered obselete. """ @@ -151,8 +153,8 @@ class GreasemonkeyManager(QObject): def load_scripts(self): """Re-read Greasemonkey scripts from disk. - The scripts are read from a 'greasemonkey' subdirectory in qutebrowser's - data directory (see `:version`). + The scripts are read from a 'greasemonkey' subdirectory in + qutebrowser's data directory (see `:version`). """ self._run_start = [] self._run_end = [] diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index dade671c8..c2c388993 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -245,10 +245,10 @@ def _init_profiles(): def inject_userscripts(): - """Register user javascript files with the global profiles.""" - # The greasemonkey metadata block support in qtwebengine only starts at 5.8 - # Otherwise have to handle injecting the scripts into the page at very - # early load, probs same place in view as the enableJS check. + """Register user JavaScript files with the global profiles.""" + # The Greasemonkey metadata block support in QtWebEngine only starts at + # Qt 5.8. With 5.7.1, we need to inject the scripts ourselves in response + # to urlChanged. if not qtutils.version_check('5.8'): return @@ -256,10 +256,7 @@ def inject_userscripts(): # just get replaced by new gm scripts like if we were injecting them # ourselves so we need to remove all gm scripts, while not removing # any other stuff that might have been added. Like the one for - # stylsheets. - # Could either use a different world for gm scripts, check for gm metadata - # values (would mean no non-gm userscripts), or check the code for - # _qute_script_id + # stylesheets. for profile in [default_profile, private_profile]: scripts = profile.scripts() for script in scripts.toList(): diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index af4476d4a..0668e3aa5 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -308,7 +308,7 @@ class WebEnginePage(QWebEnginePage): def _inject_userjs(self, url): """Inject userscripts registered for `url` into the current page.""" if qtutils.version_check('5.8'): - # Handled in webenginetab with the builtin greasemonkey + # Handled in webenginetab with the builtin Greasemonkey # support. return diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 02aa270d7..89407fcdf 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -95,7 +95,7 @@ class BrowserPage(QWebPage): """Connect userjs related signals to `frame`. Connect the signals used as triggers for injecting user - javascripts into the passed QWebFrame. + JavaScripts into the passed QWebFrame. """ log.greasemonkey.debug("Connecting to frame {} ({})" .format(frame, frame.url().toDisplayString())) @@ -299,7 +299,7 @@ class BrowserPage(QWebPage): self.error_occurred = False def _inject_userjs(self, frame): - """Inject user javascripts into the page. + """Inject user JavaScripts into the page. Args: frame: The QWebFrame to inject the user scripts into. diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index e86991040..2d36220dc 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -1,5 +1,5 @@ -(function () { - const _qute_script_id = "__gm_"+{{ scriptName | tojson }}; +(function() { + const _qute_script_id = "__gm_" + {{ scriptName | tojson }}; function GM_log(text) { console.log(text); @@ -10,7 +10,8 @@ 'scriptMetaStr': {{ scriptMeta | tojson }}, 'scriptWillUpdate': false, 'version': "0.0.1", - 'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction + // so scripts don't expect exportFunction + 'scriptHandler': 'Tampermonkey', }; function checkKey(key, funcName) { @@ -40,7 +41,7 @@ } function GM_listValues() { - let keys = []; + const keys = []; for (let i = 0; i < localStorage.length; i++) { if (localStorage.key(i).startsWith(_qute_script_id)) { keys.push(localStorage.key(i).slice(_qute_script_id.length)); @@ -59,28 +60,28 @@ details.method = details.method ? details.method.toUpperCase() : "GET"; if (!details.url) { - throw ("GM_xmlhttpRequest requires an URL."); + throw new Error("GM_xmlhttpRequest requires an URL."); } // build XMLHttpRequest object - let oXhr = new XMLHttpRequest(); + const oXhr = new XMLHttpRequest(); // run it if ("onreadystatechange" in details) { - oXhr.onreadystatechange = function () { + oXhr.onreadystatechange = function() { details.onreadystatechange(oXhr); }; } if ("onload" in details) { - oXhr.onload = function () { details.onload(oXhr) }; + oXhr.onload = function() { details.onload(oXhr); }; } if ("onerror" in details) { - oXhr.onerror = function () { details.onerror(oXhr) }; + oXhr.onerror = function () { details.onerror(oXhr); }; } oXhr.open(details.method, details.url, true); if ("headers" in details) { - for (let header in details.headers) { + for (const header in details.headers) { oXhr.setRequestHeader(header, details.headers[header]); } } @@ -93,26 +94,25 @@ } function GM_addStyle(/* String */ styles) { - let oStyle = document.createElement("style"); + const oStyle = document.createElement("style"); oStyle.setAttribute("type", "text/css"); oStyle.appendChild(document.createTextNode(styles)); - let head = document.getElementsByTagName("head")[0]; + const head = document.getElementsByTagName("head")[0]; if (head === undefined) { - document.onreadystatechange = function () { - if (document.readyState == "interactive") { + document.onreadystatechange = function() { + if (document.readyState === "interactive") { document.getElementsByTagName("head")[0].appendChild(oStyle); } - } - } - else { + }; + } else { head.appendChild(oStyle); } } const unsafeWindow = window; - //====== The actual user script source ======// + // ====== The actual user script source ====== // {{ scriptSource }} - //====== End User Script ======// + // ====== End User Script ====== // })(); diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature index 944d2606d..3ccd50efb 100644 --- a/tests/end2end/features/javascript.feature +++ b/tests/end2end/features/javascript.feature @@ -124,8 +124,8 @@ Feature: Javascript stuff And I run :tab-next Then the window sizes should be the same - Scenario: Have a greasemonkey script run at page start - When I have a greasemonkey file saved for document-start with noframes unset + Scenario: Have a GreaseMonkey script run at page start + When I have a GreaseMonkey file saved for document-start with noframes unset And I run :greasemonkey-reload And I open data/hints/iframe.html # This second reload is required in webengine < 5.8 for scripts @@ -133,15 +133,15 @@ Feature: Javascript stuff And I run :reload Then the javascript message "Script is running on /data/hints/iframe.html" should be logged - Scenario: Have a greasemonkey script running on frames - When I have a greasemonkey file saved for document-end with noframes unset + Scenario: Have a GreaseMonkey script running on frames + When I have a GreaseMonkey file saved for document-end with noframes unset And I run :greasemonkey-reload And I open data/hints/iframe.html Then the javascript message "Script is running on /data/hints/html/wrapped.html" should be logged @flaky - Scenario: Have a greasemonkey script running on noframes - When I have a greasemonkey file saved for document-end with noframes set + Scenario: Have a GreaseMonkey script running on noframes + When I have a GreaseMonkey file saved for document-end with noframes set And I run :greasemonkey-reload And I open data/hints/iframe.html Then the javascript message "Script is running on /data/hints/html/wrapped.html" should not be logged diff --git a/tests/end2end/features/test_javascript_bdd.py b/tests/end2end/features/test_javascript_bdd.py index 16896d4b5..8f69ef6d4 100644 --- a/tests/end2end/features/test_javascript_bdd.py +++ b/tests/end2end/features/test_javascript_bdd.py @@ -35,7 +35,7 @@ def check_window_sizes(quteproc): test_gm_script = r""" // ==UserScript== -// @name Qutebrowser test userscript +// @name qutebrowser test userscript // @namespace invalid.org // @include http://localhost:*/data/hints/iframe.html // @include http://localhost:*/data/hints/html/wrapped.html @@ -47,7 +47,7 @@ console.log("Script is running on " + window.location.pathname); """ -@bdd.when(bdd.parsers.parse("I have a greasemonkey file saved for {stage} " +@bdd.when(bdd.parsers.parse("I have a GreaseMonkey file saved for {stage} " "with noframes {frameset}")) def create_greasemonkey_file(quteproc, stage, frameset): script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey') diff --git a/tests/unit/javascript/test_greasemonkey.py b/tests/unit/javascript/test_greasemonkey.py index b0ba64bdf..670be416d 100644 --- a/tests/unit/javascript/test_greasemonkey.py +++ b/tests/unit/javascript/test_greasemonkey.py @@ -28,7 +28,7 @@ from qutebrowser.browser import greasemonkey test_gm_script = """ // ==UserScript== -// @name Qutebrowser test userscript +// @name qutebrowser test userscript // @namespace invalid.org // @include http://localhost:*/data/title.html // @match http://trolol* @@ -58,7 +58,7 @@ def test_all(): gm_manager = greasemonkey.GreasemonkeyManager() assert (gm_manager.all_scripts()[0].name == - "Qutebrowser test userscript") + "qutebrowser test userscript") @pytest.mark.parametrize("url, expected_matches", [ @@ -70,7 +70,7 @@ def test_all(): ('https://badhost.xxx/', 0), ]) def test_get_scripts_by_url(url, expected_matches): - """Check greasemonkey include/exclude rules work.""" + """Check Greasemonkey include/exclude rules work.""" save_script(test_gm_script, 'test.user.js') gm_manager = greasemonkey.GreasemonkeyManager() From a37ecc353cb29d5555dab019e3082a7d82d7eac7 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 19:53:58 +0100 Subject: [PATCH 280/415] Simplify for loop --- qutebrowser/browser/webengine/webenginesettings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index c2c388993..35eb4b916 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -257,6 +257,7 @@ def inject_userscripts(): # ourselves so we need to remove all gm scripts, while not removing # any other stuff that might have been added. Like the one for # stylesheets. + greasemonkey = objreg.get('greasemonkey') for profile in [default_profile, private_profile]: scripts = profile.scripts() for script in scripts.toList(): @@ -265,9 +266,7 @@ def inject_userscripts(): .format(script.name())) scripts.remove(script) - for profile in [default_profile, private_profile]: - scripts = profile.scripts() - greasemonkey = objreg.get('greasemonkey') + # Then add the new scripts. for script in greasemonkey.all_scripts(): new_script = QWebEngineScript() new_script.setWorldId(QWebEngineScript.MainWorld) From 6aafe02320d539943967896cc4901c2eafcb2f47 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 19:56:44 +0100 Subject: [PATCH 281/415] Make sure scripts are removed correctly --- qutebrowser/browser/webengine/webenginesettings.py | 5 +++-- qutebrowser/browser/webengine/webview.py | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 35eb4b916..adc7a1034 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -262,9 +262,10 @@ def inject_userscripts(): scripts = profile.scripts() for script in scripts.toList(): if script.name().startswith("GM-"): - log.greasemonkey.debug('removing script: {}' + log.greasemonkey.debug('Removing script: {}' .format(script.name())) - scripts.remove(script) + removed = scripts.remove(script) + assert removed, script.name() # Then add the new scripts. for script in greasemonkey.all_scripts(): diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index 0668e3aa5..b313fc36c 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -319,9 +319,10 @@ class WebEnginePage(QWebEnginePage): scripts = self.scripts() for script in scripts.toList(): if script.name().startswith("GM-"): - really_removed = scripts.remove(script) - log.greasemonkey.debug("Removing ({}) script: {}" - .format(really_removed, script.name())) + log.greasemonkey.debug("Removing script: {}" + .format(script.name())) + removed = scripts.remove(script) + assert removed, script.name() def _add_script(script, injection_point): new_script = QWebEngineScript() From d6039a0e348651fd4b482b325b4b3ed3c5e146b5 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 20:30:46 +0100 Subject: [PATCH 282/415] Fix markers for editor test --- tests/end2end/features/editor.feature | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 3e1be47bc..15da4a6cd 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -116,9 +116,8 @@ Feature: Opening external editors Then the javascript message "text: foobar" should be logged # Could not get signals working on Windows - @posix # There's no guarantee that the tab gets deleted... - @flaky + @posix @flaky Scenario: Spawning an editor and closing the tab When I set up a fake editor that waits And I open data/editor.html From eb90f9835fa2e159842ae16072b80109b08a977c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 20:54:14 +0100 Subject: [PATCH 283/415] Mark qute://settings test as flaky --- tests/end2end/features/qutescheme.feature | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/end2end/features/qutescheme.feature b/tests/end2end/features/qutescheme.feature index 51db7f767..1f13a8ac1 100644 --- a/tests/end2end/features/qutescheme.feature +++ b/tests/end2end/features/qutescheme.feature @@ -113,6 +113,8 @@ Feature: Special qute:// pages And I wait for "Config option changed: ignore_case *" in the log Then the option ignore_case should be set to always + # Sometimes, an unrelated value gets set + @flaky Scenario: Focusing input fields in qute://settings and entering invalid value When I open qute://settings # scroll to the right - the table does not fit in the default screen From 04d200452817e58d357627ca87aabe6ef7e6b044 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 20:58:14 +0100 Subject: [PATCH 284/415] tox: Fix eslint environment We need to set basepython there to not get InterpreterNotFound --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index e59e8b660..5b8bc05b5 100644 --- a/tox.ini +++ b/tox.ini @@ -181,6 +181,7 @@ commands = [testenv:eslint] # This is duplicated in travis_run.sh for Travis CI because we can't get tox in # the JavaScript environment easily. +basepython = python3 deps = whitelist_externals = eslint changedir = {toxinidir}/qutebrowser/javascript From 30b25da2735e597104f9f6b4fde6842261fcac86 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 6 Dec 2017 13:09:44 -0700 Subject: [PATCH 285/415] Added protocol key to field --- qutebrowser/config/configdata.yml | 4 ++++ qutebrowser/mainwindow/tabwidget.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 08c854ed3..efcc07b35 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1292,6 +1292,7 @@ tabs.title.format: - host - private - current_url + - protocol none_ok: true desc: | Format to use for the tab title. @@ -1308,6 +1309,7 @@ tabs.title.format: * `{backend}`: Either ''webkit'' or ''webengine'' * `{private}` : Indicates when private mode is enabled. * `{current_url}` : URL of the current web page. + * `{protocol}` : Internet Protocol of the current web page. tabs.title.format_pinned: default: '{index}' @@ -1324,6 +1326,7 @@ tabs.title.format_pinned: - host - private - current_url + - protocol none_ok: true desc: Format to use for the tab title for pinned tabs. The same placeholders like for `tabs.title.format` are defined. @@ -1467,6 +1470,7 @@ window.title_format: - backend - private - current_url + - protocol default: '{perc}{title}{title_sep}qutebrowser' desc: | Format to use for the window title. The same placeholders like for diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index b14466337..31615cccc 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -181,6 +181,11 @@ class TabWidget(QTabWidget): except qtutils.QtValueError: fields['current_url'] = '' + try: + fields['protocol'] = self.tab_url(idx).url(options = QUrl.RemovePath) + except qtutils.QtValueError: + fields['protocol']= '' + y = tab.scroller.pos_perc()[1] if y is None: scroll_pos = '???' From f033b228b1d94f7dbba54e552bdf7eb2870a9dd0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 20:22:03 +0100 Subject: [PATCH 286/415] Use py.path.local in save_script --- tests/unit/javascript/test_greasemonkey.py | 24 +++++++++------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/unit/javascript/test_greasemonkey.py b/tests/unit/javascript/test_greasemonkey.py index 670be416d..0f5fe476c 100644 --- a/tests/unit/javascript/test_greasemonkey.py +++ b/tests/unit/javascript/test_greasemonkey.py @@ -18,10 +18,10 @@ """Tests for qutebrowser.browser.greasemonkey.""" -import os import logging import pytest +import py.path # pylint: disable=no-name-in-module from PyQt5.QtCore import QUrl from qutebrowser.browser import greasemonkey @@ -41,20 +41,16 @@ console.log("Script is running."); pytestmark = pytest.mark.usefixtures('data_tmpdir') -def save_script(script_text, filename): - script_path = greasemonkey._scripts_dir() - try: - os.mkdir(script_path) - except FileExistsError: - pass - file_path = os.path.join(script_path, filename) - with open(file_path, 'w', encoding='utf-8') as f: - f.write(script_text) +def _save_script(script_text, filename): + # pylint: disable=no-member + file_path = py.path.local(greasemonkey._scripts_dir()) / filename + # pylint: enable=no-member + file_path.write_text(script_text, encoding='utf-8', ensure=True) def test_all(): """Test that a script gets read from file, parsed and returned.""" - save_script(test_gm_script, 'test.user.js') + _save_script(test_gm_script, 'test.user.js') gm_manager = greasemonkey.GreasemonkeyManager() assert (gm_manager.all_scripts()[0].name == @@ -71,7 +67,7 @@ def test_all(): ]) def test_get_scripts_by_url(url, expected_matches): """Check Greasemonkey include/exclude rules work.""" - save_script(test_gm_script, 'test.user.js') + _save_script(test_gm_script, 'test.user.js') gm_manager = greasemonkey.GreasemonkeyManager() scripts = gm_manager.scripts_for(QUrl(url)) @@ -81,7 +77,7 @@ def test_get_scripts_by_url(url, expected_matches): def test_no_metadata(caplog): """Run on all sites at document-end is the default.""" - save_script("var nothing = true;\n", 'nothing.user.js') + _save_script("var nothing = true;\n", 'nothing.user.js') with caplog.at_level(logging.WARNING): gm_manager = greasemonkey.GreasemonkeyManager() @@ -93,7 +89,7 @@ def test_no_metadata(caplog): def test_bad_scheme(caplog): """qute:// isn't in the list of allowed schemes.""" - save_script("var nothing = true;\n", 'nothing.user.js') + _save_script("var nothing = true;\n", 'nothing.user.js') with caplog.at_level(logging.WARNING): gm_manager = greasemonkey.GreasemonkeyManager() From 94809032a46dddbce62f279d0d9e5b85e50eb52b Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 6 Dec 2017 13:24:27 -0700 Subject: [PATCH 287/415] field[protocol] gives the right protocol] --- qutebrowser/mainwindow/tabwidget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 31615cccc..988455a08 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -182,7 +182,7 @@ class TabWidget(QTabWidget): fields['current_url'] = '' try: - fields['protocol'] = self.tab_url(idx).url(options = QUrl.RemovePath) + fields['protocol'] = self.tab_url(idx).scheme() except qtutils.QtValueError: fields['protocol']= '' From 02b24e8dfb4e7a2e65c363ba4e82708386e8e81a Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Wed, 6 Dec 2017 21:35:09 -0700 Subject: [PATCH 288/415] Update tabwidget.py --- qutebrowser/mainwindow/tabwidget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 988455a08..01b9f0728 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -184,7 +184,7 @@ class TabWidget(QTabWidget): try: fields['protocol'] = self.tab_url(idx).scheme() except qtutils.QtValueError: - fields['protocol']= '' + fields['protocol'] = '' y = tab.scroller.pos_perc()[1] if y is None: From 4d13941290f28dfe5de2ec3be3238fba41fb263b Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Wed, 6 Dec 2017 23:57:19 -0700 Subject: [PATCH 289/415] added the scheme field to FakeURL --- tests/helpers/stubs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 878c9e166..cb3c85896 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -131,11 +131,12 @@ class FakeUrl: """QUrl stub which provides .path(), isValid() and host().""" - def __init__(self, path=None, valid=True, host=None, url=None): + def __init__(self, path=None, valid=True, host=None, url=None, scheme = None): self.path = mock.Mock(return_value=path) self.isValid = mock.Mock(returl_value=valid) self.host = mock.Mock(returl_value=host) self.url = mock.Mock(return_value=url) + self.scheme = mock.Mock(return_value=scheme) class FakeNetworkReply: From d4cadcc62ea4ecd65b88aec90a27ceef8ed9fa86 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 7 Dec 2017 08:17:15 +0100 Subject: [PATCH 290/415] Add comment about @run-at [ci skip] --- qutebrowser/browser/webengine/webenginesettings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index adc7a1034..f8b54e065 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -269,6 +269,8 @@ def inject_userscripts(): # Then add the new scripts. for script in greasemonkey.all_scripts(): + # @run-at (and @include/@exclude/@match) is parsed by + # QWebEngineScript. new_script = QWebEngineScript() new_script.setWorldId(QWebEngineScript.MainWorld) new_script.setSourceCode(script.code()) From 20ac618752a6e33f7cd8ed51e6e8da8213eee29b Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Thu, 7 Dec 2017 02:04:02 -0700 Subject: [PATCH 291/415] Simplified code in get_tab_fields changed self.tab_url(idx) to url in get_tab_fields() --- qutebrowser/mainwindow/tabwidget.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 01b9f0728..9dc6cd9bf 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -165,6 +165,8 @@ class TabWidget(QTabWidget): fields['perc_raw'] = tab.progress() fields['backend'] = objects.backend.name fields['private'] = ' [Private Mode] ' if tab.private else '' + + url = self.tab_url(idx) if tab.load_status() == usertypes.LoadStatus.loading: fields['perc'] = '[{}%] '.format(tab.progress()) @@ -172,17 +174,17 @@ class TabWidget(QTabWidget): fields['perc'] = '' try: - fields['host'] = self.tab_url(idx).host() + fields['host'] = url.host() except qtutils.QtValueError: fields['host'] = '' try: - fields['current_url'] = self.tab_url(idx).url() + fields['current_url'] = url.url() except qtutils.QtValueError: fields['current_url'] = '' try: - fields['protocol'] = self.tab_url(idx).scheme() + fields['protocol'] = url.scheme() except qtutils.QtValueError: fields['protocol'] = '' From d1a00eb93446e501e97871485738b903668a89a5 Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Thu, 7 Dec 2017 02:35:34 -0700 Subject: [PATCH 292/415] Clarity on protocol field --- qutebrowser/config/configdata.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index efcc07b35..7c94d4701 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1307,9 +1307,9 @@ tabs.title.format: * `{scroll_pos}`: Page scroll position. * `{host}`: Host of the current web page. * `{backend}`: Either ''webkit'' or ''webengine'' - * `{private}` : Indicates when private mode is enabled. - * `{current_url}` : URL of the current web page. - * `{protocol}` : Internet Protocol of the current web page. + * `{private}`: Indicates when private mode is enabled. + * `{current_url}`: URL of the current web page. + * `{protocol}`: Protocol (http/https/...) of the current web page tabs.title.format_pinned: default: '{index}' From 18609f1a2444cc533c4fabdf046f3da9ff6e3278 Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Thu, 7 Dec 2017 02:36:31 -0700 Subject: [PATCH 293/415] fixed spacing on FakeURL --- tests/helpers/stubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index cb3c85896..d2c0fd746 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -131,7 +131,7 @@ class FakeUrl: """QUrl stub which provides .path(), isValid() and host().""" - def __init__(self, path=None, valid=True, host=None, url=None, scheme = None): + def __init__(self, path=None, valid=True, host=None, url=None, scheme=None): self.path = mock.Mock(return_value=path) self.isValid = mock.Mock(returl_value=valid) self.host = mock.Mock(returl_value=host) From 25526f00bf959432c2d0d40606169a50173476b3 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 7 Dec 2017 15:47:03 -0700 Subject: [PATCH 294/415] fixed catch error in tabwidget --- qutebrowser/mainwindow/tabwidget.py | 16 +++++----------- tests/helpers/stubs.py | 3 ++- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 9dc6cd9bf..28cfac0fb 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -165,8 +165,6 @@ class TabWidget(QTabWidget): fields['perc_raw'] = tab.progress() fields['backend'] = objects.backend.name fields['private'] = ' [Private Mode] ' if tab.private else '' - - url = self.tab_url(idx) if tab.load_status() == usertypes.LoadStatus.loading: fields['perc'] = '[{}%] '.format(tab.progress()) @@ -174,19 +172,15 @@ class TabWidget(QTabWidget): fields['perc'] = '' try: - fields['host'] = url.host() + url = self.tab_url(idx) except qtutils.QtValueError: fields['host'] = '' - - try: - fields['current_url'] = url.url() - except qtutils.QtValueError: fields['current_url'] = '' - - try: - fields['protocol'] = url.scheme() - except qtutils.QtValueError: fields['protocol'] = '' + else: + fields['host'] = url.host() + fields['current_url'] = url.toDisplayString() + fields['protocol'] = url.scheme() y = tab.scroller.pos_perc()[1] if y is None: diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index d2c0fd746..ede073130 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -131,7 +131,8 @@ class FakeUrl: """QUrl stub which provides .path(), isValid() and host().""" - def __init__(self, path=None, valid=True, host=None, url=None, scheme=None): + def __init__(self, path=None, valid=True, host=None, url=None, + scheme=None): self.path = mock.Mock(return_value=path) self.isValid = mock.Mock(returl_value=valid) self.host = mock.Mock(returl_value=host) From 9685eb36b69e091cc99652f4cfc4f73d331d1aab Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Thu, 7 Dec 2017 16:30:34 -0700 Subject: [PATCH 295/415] Changed FakeUrl's url command to toDisplayString --- tests/helpers/stubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index ede073130..0fd2ba0ed 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -136,7 +136,7 @@ class FakeUrl: self.path = mock.Mock(return_value=path) self.isValid = mock.Mock(returl_value=valid) self.host = mock.Mock(returl_value=host) - self.url = mock.Mock(return_value=url) + self.toDisplayString = mock.Mock(return_value=url) self.scheme = mock.Mock(return_value=scheme) From 9f9311840a2a46b25288d5ed2f432bec27802a9a Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Fri, 8 Dec 2017 16:44:53 +0000 Subject: [PATCH 296/415] Add --output-to-tab flag for :spawn. This puts the exit status, stdout, and stderr in a new tab. --- doc/help/commands.asciidoc | 3 ++- qutebrowser/browser/commands.py | 6 ++++-- qutebrowser/browser/qutescheme.py | 8 ++++++++ qutebrowser/misc/guiprocess.py | 33 ++++++++++++++++++++++++++++--- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 5d026bfca..2487b8c85 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -1148,7 +1148,7 @@ Set a mark at the current scroll position in the current tab. [[spawn]] === spawn -Syntax: +:spawn [*--userscript*] [*--verbose*] [*--detach*] 'cmdline'+ +Syntax: +:spawn [*--userscript*] [*--verbose*] [*--output-to-tab*] [*--detach*] 'cmdline'+ Spawn a command in a shell. @@ -1163,6 +1163,7 @@ Spawn a command in a shell. - `/usr/share/qutebrowser/userscripts` * +*-v*+, +*--verbose*+: Show notifications when the command started/exited. +* +*-v*+, +*--output-to-tab*+: Show stderr, stdout, and exit status in a new tab. * +*-d*+, +*--detach*+: Whether the command should be detached from qutebrowser. ==== note diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index bced4daf4..8c623aa37 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1177,7 +1177,8 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0, no_replace_variables=True) - def spawn(self, cmdline, userscript=False, verbose=False, detach=False): + def spawn(self, cmdline, userscript=False, verbose=False, + output_to_tab=False, detach=False): """Spawn a command in a shell. Args: @@ -1208,7 +1209,8 @@ class CommandDispatcher: else: cmd = os.path.expanduser(cmd) proc = guiprocess.GUIProcess(what='command', verbose=verbose, - parent=self._tabbed_browser) + parent=self._tabbed_browser, + output_to_tab=output_to_tab) if detach: proc.start_detached(cmd, args) else: diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index e6262a007..3bd2c66a4 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -42,6 +42,7 @@ from qutebrowser.misc import objects pyeval_output = ":pyeval was never called" +spawn_output = ":spawn was never called" _HANDLERS = {} @@ -268,6 +269,13 @@ def qute_pyeval(_url): return 'text/html', html +@add_handler('spawn_output') +def qute_spawn_output(_url): + """Handler for qute://spawn_output.""" + html = jinja.render('pre.html', title='spawn output', content=spawn_output) + return 'text/html', html + + @add_handler('version') @add_handler('verizon') def qute_version(_url): diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index 1adf6817e..3c2057864 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -22,9 +22,11 @@ import shlex from PyQt5.QtCore import (pyqtSlot, pyqtSignal, QObject, QProcess, - QProcessEnvironment) + QProcessEnvironment, QUrl) -from qutebrowser.utils import message, log +from qutebrowser.utils import message, log, objreg + +from qutebrowser.browser import qutescheme # A mapping of QProcess::ErrorCode's to human-readable strings. @@ -62,10 +64,11 @@ class GUIProcess(QObject): started = pyqtSignal() def __init__(self, what, *, verbose=False, additional_env=None, - parent=None): + parent=None, output_to_tab=False): super().__init__(parent) self._what = what self.verbose = verbose + self.output_to_tab = output_to_tab self._started = False self.cmd = None self.args = None @@ -96,6 +99,30 @@ class GUIProcess(QObject): self._started = False log.procs.debug("Process finished with code {}, status {}.".format( code, status)) + + stdout = None + stderr = None + + if self.output_to_tab: + stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') + stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') + + spawn_log = "" + + spawn_log += "Process finished with code {},status {}.".format( + code, status) + + if stdout: + spawn_log += "\nProcess stdout:\n" + stdout.strip() + if stderr: + spawn_log += "\nProcess stderr:\n" + stderr.strip() + + qutescheme.spawn_output = spawn_log + + tabbed_browser = objreg.get('tabbed-browser', scope='window', + window='last-focused') + tabbed_browser.openurl(QUrl('qute://spawn_output'), newtab=True) + if status == QProcess.CrashExit: message.error("{} crashed!".format(self._what.capitalize())) elif status == QProcess.NormalExit and code == 0: From 9f8dbe95e47e5ac9440b89e31c72edb79857ee0a Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Fri, 8 Dec 2017 19:00:46 +0000 Subject: [PATCH 297/415] Code review changes. This fixes the following problems found in a review: 1. Manual modification of the asciidoc has been undone. 2. --output-to-tab has been renamed to the less verbose --output. 3. spawn_output has been changed to spawn-output in the url. 4. Erroneous newline in imports has been removed. 5. output in guiprocess.py has been marked private. 6. If there is no output for either stderr or stdout, say so. 7. Missing space in a text line was added. 8. Redundant initialising of an empty string removed. --- doc/help/commands.asciidoc | 4 ++-- qutebrowser/browser/commands.py | 5 +++-- qutebrowser/browser/qutescheme.py | 4 ++-- qutebrowser/misc/guiprocess.py | 24 ++++++++++-------------- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 2487b8c85..3da81a391 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -1148,7 +1148,7 @@ Set a mark at the current scroll position in the current tab. [[spawn]] === spawn -Syntax: +:spawn [*--userscript*] [*--verbose*] [*--output-to-tab*] [*--detach*] 'cmdline'+ +Syntax: +:spawn [*--userscript*] [*--verbose*] [*--output*] [*--detach*] 'cmdline'+ Spawn a command in a shell. @@ -1163,7 +1163,7 @@ Spawn a command in a shell. - `/usr/share/qutebrowser/userscripts` * +*-v*+, +*--verbose*+: Show notifications when the command started/exited. -* +*-v*+, +*--output-to-tab*+: Show stderr, stdout, and exit status in a new tab. +* +*-o*+, +*--output*+: Whether the output should be shown in a new tab. * +*-d*+, +*--detach*+: Whether the command should be detached from qutebrowser. ==== note diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 8c623aa37..7ad73708a 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1178,7 +1178,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0, no_replace_variables=True) def spawn(self, cmdline, userscript=False, verbose=False, - output_to_tab=False, detach=False): + output=False, detach=False): """Spawn a command in a shell. Args: @@ -1189,6 +1189,7 @@ class CommandDispatcher: (or `$XDG_DATA_DIR`) - `/usr/share/qutebrowser/userscripts` verbose: Show notifications when the command started/exited. + output: Whether the output should be shown in a new tab. detach: Whether the command should be detached from qutebrowser. cmdline: The commandline to execute. """ @@ -1210,7 +1211,7 @@ class CommandDispatcher: cmd = os.path.expanduser(cmd) proc = guiprocess.GUIProcess(what='command', verbose=verbose, parent=self._tabbed_browser, - output_to_tab=output_to_tab) + output=output) if detach: proc.start_detached(cmd, args) else: diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 3bd2c66a4..32bc5806a 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -269,9 +269,9 @@ def qute_pyeval(_url): return 'text/html', html -@add_handler('spawn_output') +@add_handler('spawn-output') def qute_spawn_output(_url): - """Handler for qute://spawn_output.""" + """Handler for qute://spawn-output.""" html = jinja.render('pre.html', title='spawn output', content=spawn_output) return 'text/html', html diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index 3c2057864..32a4fbf62 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -25,7 +25,6 @@ from PyQt5.QtCore import (pyqtSlot, pyqtSignal, QObject, QProcess, QProcessEnvironment, QUrl) from qutebrowser.utils import message, log, objreg - from qutebrowser.browser import qutescheme # A mapping of QProcess::ErrorCode's to human-readable strings. @@ -50,6 +49,7 @@ class GUIProcess(QObject): cmd: The command which was started. args: A list of arguments which gets passed. verbose: Whether to show more messages. + _output: Whether to show the output in a new tab. _started: Whether the underlying process is started. _proc: The underlying QProcess. _what: What kind of thing is spawned (process/editor/userscript/...). @@ -64,11 +64,11 @@ class GUIProcess(QObject): started = pyqtSignal() def __init__(self, what, *, verbose=False, additional_env=None, - parent=None, output_to_tab=False): + parent=None, output=False): super().__init__(parent) self._what = what self.verbose = verbose - self.output_to_tab = output_to_tab + self._output = output self._started = False self.cmd = None self.args = None @@ -100,28 +100,24 @@ class GUIProcess(QObject): log.procs.debug("Process finished with code {}, status {}.".format( code, status)) - stdout = None - stderr = None - - if self.output_to_tab: + if self._output: stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') - spawn_log = "" + stderr = stderr or "(No output)" + stdout = stdout or "(No output)" - spawn_log += "Process finished with code {},status {}.".format( + spawn_log = "Process finished with code {}, status {}.".format( code, status) - if stdout: - spawn_log += "\nProcess stdout:\n" + stdout.strip() - if stderr: - spawn_log += "\nProcess stderr:\n" + stderr.strip() + spawn_log += "\nProcess stdout:\n" + stdout.strip() + spawn_log += "\nProcess stderr:\n" + stderr.strip() qutescheme.spawn_output = spawn_log tabbed_browser = objreg.get('tabbed-browser', scope='window', window='last-focused') - tabbed_browser.openurl(QUrl('qute://spawn_output'), newtab=True) + tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True) if status == QProcess.CrashExit: message.error("{} crashed!".format(self._what.capitalize())) From 038bb85a67c1115c7091f89c3f83fa55381ea647 Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Sun, 10 Dec 2017 19:12:47 +0000 Subject: [PATCH 298/415] Capture stdout and stderr always for spawn. This change makes it so that stderr and stdout is unconditionally read from for a completed process, and sent to qute://spawn-output. This allows the user to see the results of the previous process, even if they had forgotten to use --output. --- qutebrowser/misc/guiprocess.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index 32a4fbf62..9d0b50be6 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -100,21 +100,18 @@ class GUIProcess(QObject): log.procs.debug("Process finished with code {}, status {}.".format( code, status)) + stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') + stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') + + spawn_log = "Process finished with code {}, status {}.".format( + code, status) + + spawn_log += "\nProcess stdout:\n" + (stdout or "(No output)").strip() + spawn_log += "\nProcess stderr:\n" + (stderr or "(No output)").strip() + + qutescheme.spawn_output = spawn_log + if self._output: - stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') - stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') - - stderr = stderr or "(No output)" - stdout = stdout or "(No output)" - - spawn_log = "Process finished with code {}, status {}.".format( - code, status) - - spawn_log += "\nProcess stdout:\n" + stdout.strip() - spawn_log += "\nProcess stderr:\n" + stderr.strip() - - qutescheme.spawn_output = spawn_log - tabbed_browser = objreg.get('tabbed-browser', scope='window', window='last-focused') tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True) @@ -132,8 +129,6 @@ class GUIProcess(QObject): message.error("{} exited with status {}, see :messages for " "details.".format(self._what.capitalize(), code)) - stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') - stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') if stdout: log.procs.error("Process stdout:\n" + stdout.strip()) if stderr: From d32a4ea99e2d3ff691c14b6ba8ff8ec6d3d4a35a Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Sun, 10 Dec 2017 23:45:43 +0000 Subject: [PATCH 299/415] Seperate _output from guiprocess and keep window opening in spawn. This removes the extraneous variable, and makes testing easier. --- qutebrowser/browser/commands.py | 8 ++++++-- qutebrowser/misc/guiprocess.py | 33 ++++++++++++++++----------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 7ad73708a..495d48414 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1210,13 +1210,17 @@ class CommandDispatcher: else: cmd = os.path.expanduser(cmd) proc = guiprocess.GUIProcess(what='command', verbose=verbose, - parent=self._tabbed_browser, - output=output) + parent=self._tabbed_browser) if detach: proc.start_detached(cmd, args) else: proc.start(cmd, args) + if output: + tabbed_browser = objreg.get('tabbed-browser', scope='window', + window='last-focused') + tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True) + @cmdutils.register(instance='command-dispatcher', scope='window') def home(self): """Open main startpage in current tab.""" diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index 9d0b50be6..80803f53f 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -22,9 +22,9 @@ import shlex from PyQt5.QtCore import (pyqtSlot, pyqtSignal, QObject, QProcess, - QProcessEnvironment, QUrl) + QProcessEnvironment) -from qutebrowser.utils import message, log, objreg +from qutebrowser.utils import message, log from qutebrowser.browser import qutescheme # A mapping of QProcess::ErrorCode's to human-readable strings. @@ -49,7 +49,6 @@ class GUIProcess(QObject): cmd: The command which was started. args: A list of arguments which gets passed. verbose: Whether to show more messages. - _output: Whether to show the output in a new tab. _started: Whether the underlying process is started. _proc: The underlying QProcess. _what: What kind of thing is spawned (process/editor/userscript/...). @@ -64,11 +63,10 @@ class GUIProcess(QObject): started = pyqtSignal() def __init__(self, what, *, verbose=False, additional_env=None, - parent=None, output=False): + parent=None): super().__init__(parent) self._what = what self.verbose = verbose - self._output = output self._started = False self.cmd = None self.args = None @@ -103,18 +101,8 @@ class GUIProcess(QObject): stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') - spawn_log = "Process finished with code {}, status {}.".format( - code, status) - - spawn_log += "\nProcess stdout:\n" + (stdout or "(No output)").strip() - spawn_log += "\nProcess stderr:\n" + (stderr or "(No output)").strip() - - qutescheme.spawn_output = spawn_log - - if self._output: - tabbed_browser = objreg.get('tabbed-browser', scope='window', - window='last-focused') - tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True) + qutescheme.spawn_output = self.spawn_format(code, status, + stdout, stderr) if status == QProcess.CrashExit: message.error("{} crashed!".format(self._what.capitalize())) @@ -134,6 +122,17 @@ class GUIProcess(QObject): if stderr: log.procs.error("Process stderr:\n" + stderr.strip()) + def spawn_format(self, code=0, status=0, stdout="", stderr=""): + """Produce a formatted string for spawn output.""" + stdout = (stdout or "(No output)").strip() + stderr = (stderr or "(No output)").strip() + + spawn_string = ("Process finished with code {}, status {}\n" + "\nProcess stdout:\n {}" + "\nProcess stderr:\n {}").format(code, status, + stdout, stderr) + return spawn_string + @pyqtSlot() def on_started(self): """Called when the process started successfully.""" From 3b10584749c09b44e85ce45233da7d3fe0e870e9 Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Sun, 10 Dec 2017 23:46:35 +0000 Subject: [PATCH 300/415] Update tests to work with the earlier consumption of stdin etc. Note: this adds an element to vulture's whitelist that vulture mistakenly identified as unused. --- scripts/dev/run_vulture.py | 1 + tests/unit/misc/test_guiprocess.py | 30 +++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/scripts/dev/run_vulture.py b/scripts/dev/run_vulture.py index 9d21ad428..657d4b85e 100755 --- a/scripts/dev/run_vulture.py +++ b/scripts/dev/run_vulture.py @@ -82,6 +82,7 @@ def whitelist_generator(): # noqa yield 'qutebrowser.utils.jinja.Loader.get_source' yield 'qutebrowser.utils.log.QtWarningFilter.filter' yield 'qutebrowser.browser.pdfjs.is_available' + yield 'qutebrowser.misc.guiprocess.spawn_output' yield 'QEvent.posted' yield 'log_stack' # from message.py yield 'propagate' # logging.getLogger('...).propagate = False diff --git a/tests/unit/misc/test_guiprocess.py b/tests/unit/misc/test_guiprocess.py index 674c250e5..69ce3812f 100644 --- a/tests/unit/misc/test_guiprocess.py +++ b/tests/unit/misc/test_guiprocess.py @@ -19,7 +19,6 @@ """Tests for qutebrowser.misc.guiprocess.""" -import json import logging import pytest @@ -27,6 +26,7 @@ from PyQt5.QtCore import QProcess, QIODevice from qutebrowser.misc import guiprocess from qutebrowser.utils import usertypes +from qutebrowser.browser import qutescheme @pytest.fixture() @@ -60,7 +60,7 @@ def test_start(proc, qtbot, message_mock, py_proc): proc.start(*argv) assert not message_mock.messages - assert bytes(proc._proc.readAll()).rstrip() == b'test' + assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") def test_start_verbose(proc, qtbot, message_mock, py_proc): @@ -77,7 +77,24 @@ def test_start_verbose(proc, qtbot, message_mock, py_proc): assert msgs[1].level == usertypes.MessageLevel.info assert msgs[0].text.startswith("Executing:") assert msgs[1].text == "Testprocess exited successfully." - assert bytes(proc._proc.readAll()).rstrip() == b'test' + assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") + + +def test_start_output(proc, qtbot, message_mock, py_proc): + """Test starting a process verbosely.""" + proc.verbose = True + + with qtbot.waitSignals([proc.started, proc.finished], timeout=10000, + order='strict'): + argv = py_proc("import sys; print('test'); sys.exit(0)") + proc.start(*argv) + + msgs = message_mock.messages + assert msgs[0].level == usertypes.MessageLevel.info + assert msgs[1].level == usertypes.MessageLevel.info + assert msgs[0].text.startswith("Executing:") + assert msgs[1].text == "Testprocess exited successfully." + assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") def test_start_env(monkeypatch, qtbot, py_proc): @@ -99,10 +116,9 @@ def test_start_env(monkeypatch, qtbot, py_proc): order='strict'): proc.start(*argv) - data = bytes(proc._proc.readAll()).decode('utf-8') - ret_env = json.loads(data) - assert 'QUTEBROWSER_TEST_1' in ret_env - assert 'QUTEBROWSER_TEST_2' in ret_env + data = qutescheme.spawn_output + assert 'QUTEBROWSER_TEST_1' in data + assert 'QUTEBROWSER_TEST_2' in data @pytest.mark.qt_log_ignore('QIODevice::read.*: WriteOnly device') From 2a8b74cbec45d70dd0964384779ba69e1348bd28 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 07:10:17 +0100 Subject: [PATCH 301/415] Get rid of FakeUrl stub We can just use a real QUrl... --- tests/helpers/stubs.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 0fd2ba0ed..3f6a23958 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -24,7 +24,7 @@ from unittest import mock import attr -from PyQt5.QtCore import pyqtSignal, QPoint, QProcess, QObject +from PyQt5.QtCore import pyqtSignal, QPoint, QProcess, QObject, QUrl from PyQt5.QtNetwork import (QNetworkRequest, QAbstractNetworkCache, QNetworkCacheMetaData) from PyQt5.QtWidgets import QCommonStyle, QLineEdit, QWidget, QTabBar @@ -127,19 +127,6 @@ class FakeQApplication: self.activeWindow = lambda: active_window -class FakeUrl: - - """QUrl stub which provides .path(), isValid() and host().""" - - def __init__(self, path=None, valid=True, host=None, url=None, - scheme=None): - self.path = mock.Mock(return_value=path) - self.isValid = mock.Mock(returl_value=valid) - self.host = mock.Mock(returl_value=host) - self.toDisplayString = mock.Mock(return_value=url) - self.scheme = mock.Mock(return_value=scheme) - - class FakeNetworkReply: """QNetworkReply stub which provides a Content-Disposition header.""" @@ -150,7 +137,7 @@ class FakeNetworkReply: def __init__(self, headers=None, url=None): if url is None: - url = FakeUrl() + url = QUrl() if headers is None: self.headers = {} else: @@ -246,7 +233,7 @@ class FakeWebTab(browsertab.AbstractTab): """Fake AbstractTab to use in tests.""" - def __init__(self, url=FakeUrl(), title='', tab_id=0, *, + def __init__(self, url=QUrl(), title='', tab_id=0, *, scroll_pos_perc=(0, 0), load_status=usertypes.LoadStatus.success, progress=0, can_go_back=None, can_go_forward=None): From 444f0a36dfba2f54a279b0ca8dffd2c601211666 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 07:12:45 +0100 Subject: [PATCH 302/415] Update docs --- doc/changelog.asciidoc | 1 + doc/help/settings.asciidoc | 5 +++-- qutebrowser/config/configdata.yml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 16075db27..4192ee054 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -56,6 +56,7 @@ Added - New `session.lazy_restore` setting which allows to not load pages immediately when restoring a session. - New `hist_importer.py` script to import history from Firefox/Chromium. +- New `{protocol}` replacement for `tabs.title.format` and friends. Changed ~~~~~~~ diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 967f6c4c6..28480486f 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -2913,8 +2913,9 @@ The following placeholders are defined: * `{scroll_pos}`: Page scroll position. * `{host}`: Host of the current web page. * `{backend}`: Either ''webkit'' or ''webengine'' -* `{private}` : Indicates when private mode is enabled. -* `{current_url}` : URL of the current web page. +* `{private}`: Indicates when private mode is enabled. +* `{current_url}`: URL of the current web page. +* `{protocol}`: Protocol (http/https/...) of the current web page. Type: <> diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index d96718d78..1318e8979 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1317,7 +1317,7 @@ tabs.title.format: * `{backend}`: Either ''webkit'' or ''webengine'' * `{private}`: Indicates when private mode is enabled. * `{current_url}`: URL of the current web page. - * `{protocol}`: Protocol (http/https/...) of the current web page + * `{protocol}`: Protocol (http/https/...) of the current web page. tabs.title.format_pinned: default: '{index}' From f7a94b946f02dae81d32616ed3467ca51314aec1 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 08:12:13 +0100 Subject: [PATCH 303/415] Add changelog for font size change See 22f3fade24dbe6b3bdf737c8395360ca9e6b203b --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 4192ee054..fddb3f6a5 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -97,6 +97,7 @@ Changed - Empty categories are now hidden in the `:open` completion. - Search terms for URLs and titles can now be mixed when filtering the completion. +- The default font size for the UI got bumped up from 8pt to 10pt. Fixed ~~~~~ From 6a7d2f4275e18b289cdc41cab6e06984bea2d271 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 09:14:26 +0100 Subject: [PATCH 304/415] Remove dead code QUrl.path() never returns None --- qutebrowser/browser/webkit/http.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qutebrowser/browser/webkit/http.py b/qutebrowser/browser/webkit/http.py index 08cad7a44..7f5173943 100644 --- a/qutebrowser/browser/webkit/http.py +++ b/qutebrowser/browser/webkit/http.py @@ -57,9 +57,7 @@ def parse_content_disposition(reply): is_inline = content_disposition.is_inline() # Then try to get filename from url if not filename: - path = reply.url().path() - if path is not None: - filename = path.rstrip('/') + filename = reply.url().path().rstrip('/') # If that fails as well, use a fallback if not filename: filename = 'qutebrowser-download' From 72d847d687ad00effb73d77e8b0b395f34bdcdf0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 09:36:27 +0100 Subject: [PATCH 305/415] travis: Use newer macOS image --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0ea8218a6..251842d06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: env: TESTENV=py36-pyqt59-cov - os: osx env: TESTENV=py36 OSX=sierra - osx_image: xcode8.3 + osx_image: xcode9.2 language: generic # https://github.com/qutebrowser/qutebrowser/issues/2013 # - os: osx From 8909e03f1cea998b6e997486e2541bce3d94d71b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 10 Dec 2017 15:06:45 -0500 Subject: [PATCH 306/415] Match url completion terms in any order. Perviously, 'foo bar' would match 'foo/bar' but not 'bar/foo'. Now it will match both, using a query with a WHERE clause like: WHERE ((url || title) like '%foo%' AND (url || title) like '%bar%') This does not seem to change the performance benchmark. However, it does create a new query for every character added rather than re-running the same query with different parameters. We could re-use queries if we maintained a list like self._queries=[1_arg_query, 2_arg_query, ...]. However, it isn't clear that such a complexity would be necessary. Resolves #1651. --- qutebrowser/completion/models/histcategory.py | 42 ++++++++++--------- tests/unit/completion/test_histcategory.py | 2 +- tests/unit/completion/test_models.py | 4 +- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index fe89dc79b..03ddb5ff8 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -37,21 +37,6 @@ class HistoryCategory(QSqlQueryModel): super().__init__(parent=parent) self.name = "History" - # replace ' in timestamp-format to avoid breaking the query - timestamp_format = config.val.completion.timestamp_format - timefmt = ("strftime('{}', last_atime, 'unixepoch', 'localtime')" - .format(timestamp_format.replace("'", "`"))) - - self._query = sql.Query(' '.join([ - "SELECT url, title, {}".format(timefmt), - "FROM CompletionHistory", - # the incoming pattern will have literal % and _ escaped with '\' - # we need to tell sql to treat '\' as an escape character - "WHERE ((url || title) LIKE :pat escape '\\')", - self._atime_expr(), - "ORDER BY last_atime DESC", - ]), forward_only=False) - # advertise that this model filters by URL and title self.columns_to_filter = [0, 1] self.delete_func = delete_func @@ -86,11 +71,30 @@ class HistoryCategory(QSqlQueryModel): # escape to treat a user input % or _ as a literal, not a wildcard pattern = pattern.replace('%', '\\%') pattern = pattern.replace('_', '\\_') - # treat spaces as wildcards to match any of the typed words - pattern = re.sub(r' +', '%', pattern) - pattern = '%{}%'.format(pattern) + words = ['%{}%'.format(w) for w in pattern.split(' ')] + + wheres = ' AND '.join([ + "(url || title) LIKE :pat{} escape '\\'".format(i) + for i in range(len(words))]) + + # replace ' in timestamp-format to avoid breaking the query + timestamp_format = config.val.completion.timestamp_format + timefmt = ("strftime('{}', last_atime, 'unixepoch', 'localtime')" + .format(timestamp_format.replace("'", "`"))) + + self._query = sql.Query(' '.join([ + "SELECT url, title, {}".format(timefmt), + "FROM CompletionHistory", + # the incoming pattern will have literal % and _ escaped with '\' + # we need to tell sql to treat '\' as an escape character + 'WHERE ({})'.format(wheres), + self._atime_expr(), + "ORDER BY last_atime DESC", + ]), forward_only=False) + with debug.log_time('sql', 'Running completion query'): - self._query.run(pat=pattern) + self._query.run(**{ + 'pat{}'.format(i): w for i, w in enumerate(words)}) self.setQuery(self._query) def removeRows(self, row, _count, _parent=None): diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index b87eb6ac2..8458c3311 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -61,7 +61,7 @@ def hist(init_sql, config_stub): ('foo bar', [('foo', ''), ('bar foo', ''), ('xfooyybarz', '')], - [('xfooyybarz', '')]), + [('bar foo', ''), ('xfooyybarz', '')]), ('foo%bar', [('foo%bar', ''), ('foo bar', ''), ('foobar', '')], diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 8879f3201..c4d224dcc 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -414,12 +414,12 @@ def test_url_completion_no_bookmarks(qtmodeltester, web_history_populated, ('example.com', 'Site Title', 'am', 1), ('example.com', 'Site Title', 'com', 1), ('example.com', 'Site Title', 'ex com', 1), - ('example.com', 'Site Title', 'com ex', 0), + ('example.com', 'Site Title', 'com ex', 1), ('example.com', 'Site Title', 'ex foo', 0), ('example.com', 'Site Title', 'foo com', 0), ('example.com', 'Site Title', 'exm', 0), ('example.com', 'Site Title', 'Si Ti', 1), - ('example.com', 'Site Title', 'Ti Si', 0), + ('example.com', 'Site Title', 'Ti Si', 1), ('example.com', '', 'foo', 0), ('foo_bar', '', '_', 1), ('foobar', '', '_', 0), From a2bcd68d5628bcc1f0df47dd965bedf302855729 Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Mon, 11 Dec 2017 13:35:39 +0000 Subject: [PATCH 307/415] Code review changes. This fixes whitespace and alignment issues, and removes a stray test. --- qutebrowser/browser/commands.py | 4 ++-- qutebrowser/misc/guiprocess.py | 6 +++--- tests/unit/misc/test_guiprocess.py | 21 ++------------------- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 495d48414..3075a24da 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1178,7 +1178,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0, no_replace_variables=True) def spawn(self, cmdline, userscript=False, verbose=False, - output=False, detach=False): + output=False, detach=False): """Spawn a command in a shell. Args: @@ -1218,7 +1218,7 @@ class CommandDispatcher: if output: tabbed_browser = objreg.get('tabbed-browser', scope='window', - window='last-focused') + window='last-focused') tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True) @cmdutils.register(instance='command-dispatcher', scope='window') diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index 80803f53f..4b74a512d 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -101,7 +101,7 @@ class GUIProcess(QObject): stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') - qutescheme.spawn_output = self.spawn_format(code, status, + qutescheme.spawn_output = self._spawn_format(code, status, stdout, stderr) if status == QProcess.CrashExit: @@ -122,7 +122,7 @@ class GUIProcess(QObject): if stderr: log.procs.error("Process stderr:\n" + stderr.strip()) - def spawn_format(self, code=0, status=0, stdout="", stderr=""): + def _spawn_format(self, code=0, status=0, stdout="", stderr=""): """Produce a formatted string for spawn output.""" stdout = (stdout or "(No output)").strip() stderr = (stderr or "(No output)").strip() @@ -130,7 +130,7 @@ class GUIProcess(QObject): spawn_string = ("Process finished with code {}, status {}\n" "\nProcess stdout:\n {}" "\nProcess stderr:\n {}").format(code, status, - stdout, stderr) + stdout, stderr) return spawn_string @pyqtSlot() diff --git a/tests/unit/misc/test_guiprocess.py b/tests/unit/misc/test_guiprocess.py index 69ce3812f..25e46476e 100644 --- a/tests/unit/misc/test_guiprocess.py +++ b/tests/unit/misc/test_guiprocess.py @@ -60,7 +60,7 @@ def test_start(proc, qtbot, message_mock, py_proc): proc.start(*argv) assert not message_mock.messages - assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") + assert qutescheme.spawn_output == proc._spawn_format(stdout="test") def test_start_verbose(proc, qtbot, message_mock, py_proc): @@ -77,24 +77,7 @@ def test_start_verbose(proc, qtbot, message_mock, py_proc): assert msgs[1].level == usertypes.MessageLevel.info assert msgs[0].text.startswith("Executing:") assert msgs[1].text == "Testprocess exited successfully." - assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") - - -def test_start_output(proc, qtbot, message_mock, py_proc): - """Test starting a process verbosely.""" - proc.verbose = True - - with qtbot.waitSignals([proc.started, proc.finished], timeout=10000, - order='strict'): - argv = py_proc("import sys; print('test'); sys.exit(0)") - proc.start(*argv) - - msgs = message_mock.messages - assert msgs[0].level == usertypes.MessageLevel.info - assert msgs[1].level == usertypes.MessageLevel.info - assert msgs[0].text.startswith("Executing:") - assert msgs[1].text == "Testprocess exited successfully." - assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") + assert qutescheme.spawn_output == proc._spawn_format(stdout="test") def test_start_env(monkeypatch, qtbot, py_proc): From dd7a082265d4e15567c31bf61d1baec0fb232892 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:12 +0100 Subject: [PATCH 308/415] Update codecov from 2.0.9 to 2.0.10 --- misc/requirements/requirements-codecov.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-codecov.txt b/misc/requirements/requirements-codecov.txt index 31c319c39..6601cfb12 100644 --- a/misc/requirements/requirements-codecov.txt +++ b/misc/requirements/requirements-codecov.txt @@ -2,7 +2,7 @@ certifi==2017.11.5 chardet==3.0.4 -codecov==2.0.9 +codecov==2.0.10 coverage==4.4.2 idna==2.6 requests==2.18.4 From 519dc6a7c9f890f2377fc71d716f7d0ddfa4b6ce Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:13 +0100 Subject: [PATCH 309/415] Update setuptools from 38.2.3 to 38.2.4 --- misc/requirements/requirements-pip.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pip.txt b/misc/requirements/requirements-pip.txt index 8ca5a867e..b7914cac5 100644 --- a/misc/requirements/requirements-pip.txt +++ b/misc/requirements/requirements-pip.txt @@ -3,6 +3,6 @@ appdirs==1.4.3 packaging==16.8 pyparsing==2.2.0 -setuptools==38.2.3 +setuptools==38.2.4 six==1.11.0 wheel==0.30.0 From 713a2ef2c169a19db1ab6f7c97e60c2e1d2e2806 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:15 +0100 Subject: [PATCH 310/415] Update pylint from 1.7.4 to 1.7.5 --- misc/requirements/requirements-pylint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pylint.txt b/misc/requirements/requirements-pylint.txt index cab15c497..346fa4227 100644 --- a/misc/requirements/requirements-pylint.txt +++ b/misc/requirements/requirements-pylint.txt @@ -8,7 +8,7 @@ idna==2.6 isort==4.2.15 lazy-object-proxy==1.3.1 mccabe==0.6.1 -pylint==1.7.4 +pylint==1.7.5 ./scripts/dev/pylint_checkers requests==2.18.4 six==1.11.0 From 5d8e3a969f59c0484bb7f45087166e728b4467d1 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:16 +0100 Subject: [PATCH 311/415] Update cheroot from 5.10.0 to 6.0.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 46905f497..62ea92c3c 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -2,7 +2,7 @@ attrs==17.3.0 beautifulsoup4==4.6.0 -cheroot==5.10.0 +cheroot==6.0.0 click==6.7 # colorama==0.3.9 coverage==4.4.2 From 22fe42d38ea76f261aea69e8c8d961517af18b03 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:18 +0100 Subject: [PATCH 312/415] Update hypothesis from 3.40.1 to 3.42.1 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 62ea92c3c..439267282 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -11,7 +11,7 @@ fields==5.0.0 Flask==0.12.2 glob2==0.6 hunter==2.0.2 -hypothesis==3.40.1 +hypothesis==3.42.1 itsdangerous==0.24 # Jinja2==2.10 Mako==1.0.7 From d57a81a3d347975c7cb16a6194261ae320a9b5ad Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:19 +0100 Subject: [PATCH 313/415] Update pytest from 3.3.0 to 3.3.1 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 439267282..bfb8a5ea3 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -21,7 +21,7 @@ parse-type==0.4.2 pluggy==0.6.0 py==1.5.2 py-cpuinfo==3.3.0 -pytest==3.3.0 +pytest==3.3.1 pytest-bdd==2.19.0 pytest-benchmark==3.1.1 pytest-cov==2.5.1 From d114deac70d50b843291b9d3cfafe122e7afef34 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:21 +0100 Subject: [PATCH 314/415] Update werkzeug from 0.12.2 to 0.13 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index bfb8a5ea3..bf80acee7 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -36,4 +36,4 @@ pytest-xvfb==1.0.0 PyVirtualDisplay==0.2.1 six==1.11.0 vulture==0.26 -Werkzeug==0.12.2 +Werkzeug==0.13 From 481dec067dda7bf6f5a8df783568e7f7ee14923f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 17:38:12 +0100 Subject: [PATCH 315/415] Don't override background-color for qutebrowser pages Fixes #3381 --- qutebrowser/html/base.html | 1 - 1 file changed, 1 deletion(-) diff --git a/qutebrowser/html/base.html b/qutebrowser/html/base.html index f21eb227b..6db182908 100644 --- a/qutebrowser/html/base.html +++ b/qutebrowser/html/base.html @@ -10,7 +10,6 @@ vim: ft=html fileencoding=utf-8 sts=4 sw=4 et: