From 8cb6b832d1e1311ee4404a44ad2f471eaa00c884 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 00:24:59 -0400 Subject: [PATCH 001/322] script to import history data from other browsers --- scripts/hist_importer.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 scripts/hist_importer.py diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py new file mode 100644 index 000000000..e69de29bb From c6d140a40a77345c484bc8557f84f97d33539bfe Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 00:26:47 -0400 Subject: [PATCH 002/322] adding script to import history data from other browsers --- scripts/hist_importer.py | 135 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index e69de29bb..5f4ead361 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# This file is part of qutebrowser. + +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + + +''' +Tool to import browser history data from other browsers. Although, safari +support is still on the way. +''' + + +import argparse +import sqlite3 +import sys + + +def parser(): + """Parse command line arguments.""" + description = 'This program is meant to extract browser history from your'\ + 'previous browser and import them into qutebrowser.' + epilog = 'Databases:\n\tQute: Is named "history.sqlite" and can be found '\ + 'at your --basedir. In order to find where your basedir is you '\ + 'can run ":open qute:version" inside qutebrowser.'\ + '\n\tFirerox: Is named "places.sqlite", and can be found at your'\ + 'system\'s profile folder. Check this link for where it is locat'\ + 'ed: http://kb.mozillazine.org/Profile_folder'\ + '\n\tChrome: Is named "History", and can be found at the respec'\ + 'tive User Data Directory. Check this link for where it is locat'\ + 'ed: https://chromium.googlesource.com/chromium/src/+/master/'\ + 'docs/user_data_dir.md\n\n'\ + 'Example: $this_script.py -b firefox -s /Firefox/Profile/places.'\ + 'sqlite -d /qutebrowser/data/history.sqlite' + parser = argparse.ArgumentParser( + description=description, epilog=epilog, + formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument('-b', '--browser', dest='browser', required=True, + type=str, help='Browsers: {firefox, chrome, safari}') + parser.add_argument('-s', '--source', dest='source', required=True, + type=str, help='Source: fullpath to the sqlite data' + 'base file from the source browser.') + parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, + help='Destination: The fullpath to the qutebrowser ' + 'sqlite database') + return parser.parse_args() + + +def open_db(db): + """Open connection with database.""" + try: + conn = sqlite3.connect(db) + return conn + except Exception as e: + print('Error: {}'.format(e)) + raise('Error: There was some error trying to to connect with the [{}]' + 'database. Verify if the filepath is correct or is being used.'. + format(db)) + + +def extract(source, query): + """Performs extraction of (datetime,url,title) from source.""" + try: + conn = open_db(source) + cursor = conn.cursor() + cursor.execute(query) + history = cursor.fetchall() + conn.close() + return history + except Exception as e: + # print('Error: {}'.format(e)) + print(type(source)) + raise('Error: There was some error trying to to connect with the [{}]' + 'database. Verify if the filepath is correct or is being used.'. + format(str(source))) + + +def clean(history): + """Receives a list of records:(datetime,url,title). And clean all records + in place, that has a NULL/None datetime attribute. Otherwise Qutebrowser + will throw errors.""" + nulls = [record for record in history if record[0] is None] + for null_datetime in nulls: + history.remove(null_datetime) + return history + + +def insert_qb(history, dest): + conn = open_db(dest) + cursor = conn.cursor() + cursor.executemany( + 'INSERT INTO History (url,title,atime) VALUES (?,?,?)', history + ) + conn.commit() + conn.close() + + +def main(): + args = parser() + browser = args.browser.lower() + source, dest = args.source, args.dest + query = { + 'firefox': 'select url,title,last_visit_date/1000000 as date ' + 'from moz_places', + 'chrome': 'select url,title,last_visit_time/10000000 as date ' + 'from urls', + 'safari': None + } + if browser not in query: + sys.exit('Sorry, the selected browser: "{}" is not supported.'.format( + browser)) + else: + if browser == 'safari': + print('Sorry, currently we do not support this browser.') + sys.exit(1) + history = extract(source, query[browser]) + history = clean(history) + insert_qb(history, dest) + + +if __name__ == "__main__": + main() From 4dc232f259d4a24da075e584998998873e58c221 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 13:54:24 -0400 Subject: [PATCH 003/322] pylint fixes --- scripts/hist_importer.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 5f4ead361..44a76bd8a 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -17,10 +17,8 @@ # along with qutebrowser. If not, see . -''' -Tool to import browser history data from other browsers. Although, safari -support is still on the way. -''' +'''Tool to import browser history data from other browsers. Although, safari +support is still on the way.''' import argparse @@ -44,31 +42,31 @@ def parser(): 'docs/user_data_dir.md\n\n'\ 'Example: $this_script.py -b firefox -s /Firefox/Profile/places.'\ 'sqlite -d /qutebrowser/data/history.sqlite' - parser = argparse.ArgumentParser( + parsed = argparse.ArgumentParser( description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter ) - parser.add_argument('-b', '--browser', dest='browser', required=True, + parsed.add_argument('-b', '--browser', dest='browser', required=True, type=str, help='Browsers: {firefox, chrome, safari}') - parser.add_argument('-s', '--source', dest='source', required=True, + parsed.add_argument('-s', '--source', dest='source', required=True, type=str, help='Source: fullpath to the sqlite data' 'base file from the source browser.') - parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, + parsed.add_argument('-d', '--dest', dest='dest', required=True, type=str, help='Destination: The fullpath to the qutebrowser ' 'sqlite database') - return parser.parse_args() + return parsed.parse_args() -def open_db(db): +def open_db(data_base): """Open connection with database.""" try: - conn = sqlite3.connect(db) + conn = sqlite3.connect(data_base) return conn - except Exception as e: - print('Error: {}'.format(e)) + except Exception as any_e: + print('Error: {}'.format(any_e)) raise('Error: There was some error trying to to connect with the [{}]' 'database. Verify if the filepath is correct or is being used.'. - format(db)) + format(data_base)) def extract(source, query): @@ -80,8 +78,8 @@ def extract(source, query): history = cursor.fetchall() conn.close() return history - except Exception as e: - # print('Error: {}'.format(e)) + except Exception as any_e: + print('Error: {}'.format(any_e)) print(type(source)) raise('Error: There was some error trying to to connect with the [{}]' 'database. Verify if the filepath is correct or is being used.'. @@ -99,6 +97,8 @@ def clean(history): def insert_qb(history, dest): + """Given a list of records in history and a dest db, insert all records in + the dest db.""" conn = open_db(dest) cursor = conn.cursor() cursor.executemany( @@ -109,19 +109,20 @@ def insert_qb(history, dest): def main(): + """Main control flux of the script.""" args = parser() browser = args.browser.lower() source, dest = args.source, args.dest query = { 'firefox': 'select url,title,last_visit_date/1000000 as date ' - 'from moz_places', + 'from moz_places', 'chrome': 'select url,title,last_visit_time/10000000 as date ' - 'from urls', + 'from urls', 'safari': None } if browser not in query: sys.exit('Sorry, the selected browser: "{}" is not supported.'.format( - browser)) + browser)) else: if browser == 'safari': print('Sorry, currently we do not support this browser.') From 665a76561ecf405783f13beca694085a0803e479 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 2 Oct 2017 22:50:52 -0400 Subject: [PATCH 004/322] add insertions to ComandHistory table as well --- scripts/hist_importer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 44a76bd8a..8df16384a 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -104,6 +104,10 @@ def insert_qb(history, dest): cursor.executemany( 'INSERT INTO History (url,title,atime) VALUES (?,?,?)', history ) + cursor.executemany( + 'INSERT INTO CompletionHistory (url,title,last_atime) VALUES (?,?,?)', + history + ) conn.commit() conn.close() From 92f9a8503ee90cba87bc11b9aa983e8c8069c497 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Tue, 3 Oct 2017 01:55:31 -0400 Subject: [PATCH 005/322] add required redirect (url,title,atime,redirect) --- scripts/hist_importer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 8df16384a..d96284879 100644 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -89,10 +89,14 @@ def extract(source, query): def clean(history): """Receives a list of records:(datetime,url,title). And clean all records in place, that has a NULL/None datetime attribute. Otherwise Qutebrowser - will throw errors.""" + will throw errors. Also, will add a 4rth attribute of '0' for the redirect + field in history.sqlite in qutebrowser.""" nulls = [record for record in history if record[0] is None] for null_datetime in nulls: history.remove(null_datetime) + history = [list(record) for record in history] + for record in history: + record.append('0') return history @@ -102,12 +106,10 @@ def insert_qb(history, dest): conn = open_db(dest) cursor = conn.cursor() cursor.executemany( - 'INSERT INTO History (url,title,atime) VALUES (?,?,?)', history - ) - cursor.executemany( - 'INSERT INTO CompletionHistory (url,title,last_atime) VALUES (?,?,?)', + 'INSERT INTO History (url,title,atime,redirect) VALUES (?,?,?,?)', history ) + cursor.execute('DROP TABLE CompletionHistory') conn.commit() conn.close() From 96599b96846aa108698946eb7d49ad7dc33581f0 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Fri, 17 Nov 2017 02:38:56 -0300 Subject: [PATCH 006/322] revisions set by The Compiler --- scripts/hist_importer.py | 95 ++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 48 deletions(-) mode change 100644 => 100755 scripts/hist_importer.py diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py old mode 100644 new mode 100755 index d96284879..f27b4e267 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: -# This file is part of qutebrowser. +# Copyright 2017 Florian Bruhin (The Compiler) +# Copyright 2014-2017 Josefson Souza +# This file is part of qutebrowser. +# # qutebrowser is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -17,60 +20,60 @@ # along with qutebrowser. If not, see . -'''Tool to import browser history data from other browsers. Although, safari -support is still on the way.''' +"""Tool to import browser history from other browsers.""" import argparse import sqlite3 import sys +import os -def parser(): +def parse(): """Parse command line arguments.""" - description = 'This program is meant to extract browser history from your'\ - 'previous browser and import them into qutebrowser.' - epilog = 'Databases:\n\tQute: Is named "history.sqlite" and can be found '\ - 'at your --basedir. In order to find where your basedir is you '\ - 'can run ":open qute:version" inside qutebrowser.'\ - '\n\tFirerox: Is named "places.sqlite", and can be found at your'\ - 'system\'s profile folder. Check this link for where it is locat'\ - 'ed: http://kb.mozillazine.org/Profile_folder'\ - '\n\tChrome: Is named "History", and can be found at the respec'\ - 'tive User Data Directory. Check this link for where it is locat'\ - 'ed: https://chromium.googlesource.com/chromium/src/+/master/'\ - 'docs/user_data_dir.md\n\n'\ - 'Example: $this_script.py -b firefox -s /Firefox/Profile/places.'\ - 'sqlite -d /qutebrowser/data/history.sqlite' - parsed = argparse.ArgumentParser( + description = ("This program is meant to extract browser history from your" + "previous browser and import them into qutebrowser.") + epilog = ("Databases:\n\tQutebrowser: Is named 'history.sqlite' and can be" + " found at your --basedir. In order to find where your basedir" + " is you can run ':open qute:version' inside qutebrowser." + "\n\tFirerox: Is named 'places.sqlite', and can be found at your" + "system\"s profile folder. Check this link for where it is locat" + "ed: http://kb.mozillazine.org/Profile_folder" + "\n\tChrome: Is named 'History', and can be found at the respec" + "tive User Data Directory. Check this link for where it is locat" + "ed: https://chromium.googlesource.com/chromium/src/+/master/" + "docs/user_data_dir.md\n\n" + "Example: hist_importer.py -b firefox -s /Firefox/Profile/" + "places.sqlite -d /qutebrowser/data/history.sqlite") + parser = argparse.ArgumentParser( description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter ) - parsed.add_argument('-b', '--browser', dest='browser', required=True, - type=str, help='Browsers: {firefox, chrome, safari}') - parsed.add_argument('-s', '--source', dest='source', required=True, - type=str, help='Source: fullpath to the sqlite data' + parser.add_argument('-b', '--browser', dest='browser', required=True, + type=str, help='Browsers: {firefox, chrome}') + parser.add_argument('-s', '--source', dest='source', required=True, + type=str, help='Source: Full path to the sqlite data' 'base file from the source browser.') - parsed.add_argument('-d', '--dest', dest='dest', required=True, type=str, - help='Destination: The fullpath to the qutebrowser ' + parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, + help='Destination: The full path to the qutebrowser ' 'sqlite database') - return parsed.parse_args() + return parser.parse_args() def open_db(data_base): """Open connection with database.""" - try: + if os.path.isfile(data_base): conn = sqlite3.connect(data_base) return conn - except Exception as any_e: - print('Error: {}'.format(any_e)) - raise('Error: There was some error trying to to connect with the [{}]' - 'database. Verify if the filepath is correct or is being used.'. - format(data_base)) + else: + raise sys.exit('\nDataBaseNotFound: There was some error trying to to' + ' connect with the [{}] database. Verify if the' + ' filepath is correct or is being used.' + .format(data_base)) def extract(source, query): - """Performs extraction of (datetime,url,title) from source.""" + """Extracts (datetime,url,title) from source database.""" try: conn = open_db(source) cursor = conn.cursor() @@ -78,18 +81,17 @@ def extract(source, query): history = cursor.fetchall() conn.close() return history - except Exception as any_e: - print('Error: {}'.format(any_e)) - print(type(source)) - raise('Error: There was some error trying to to connect with the [{}]' - 'database. Verify if the filepath is correct or is being used.'. - format(str(source))) + except sqlite3.OperationalError as op_e: + print('\nCould not perform queries into the source database: {}' + '\nBrowser version is not supported as it have a different sql' + ' schema.'.format(op_e)) def clean(history): - """Receives a list of records:(datetime,url,title). And clean all records - in place, that has a NULL/None datetime attribute. Otherwise Qutebrowser - will throw errors. Also, will add a 4rth attribute of '0' for the redirect + """Clean up records from source database. + Receives a list of records:(datetime,url,title). And clean all records + in place, that has a NULL/None datetime attribute. Otherwise qutebrowser + will throw errors. Also, will add a 4th attribute of '0' for the redirect field in history.sqlite in qutebrowser.""" nulls = [record for record in history if record[0] is None] for null_datetime in nulls: @@ -101,7 +103,8 @@ def clean(history): def insert_qb(history, dest): - """Given a list of records in history and a dest db, insert all records in + """Insert history into dest database + Given a list of records in history and a dest db, insert all records in the dest db.""" conn = open_db(dest) cursor = conn.cursor() @@ -116,7 +119,7 @@ def insert_qb(history, dest): def main(): """Main control flux of the script.""" - args = parser() + args = parse() browser = args.browser.lower() source, dest = args.source, args.dest query = { @@ -124,15 +127,11 @@ def main(): 'from moz_places', 'chrome': 'select url,title,last_visit_time/10000000 as date ' 'from urls', - 'safari': None } if browser not in query: sys.exit('Sorry, the selected browser: "{}" is not supported.'.format( browser)) else: - if browser == 'safari': - print('Sorry, currently we do not support this browser.') - sys.exit(1) history = extract(source, query[browser]) history = clean(history) insert_qb(history, dest) From 3131d3d3bc8acf7ba40029ffcfe59f2454d0bfbc Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Fri, 17 Nov 2017 11:48:34 -0300 Subject: [PATCH 007/322] Flake8 warnings pointed by travis. --- scripts/hist_importer.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index f27b4e267..79c723c58 100755 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -73,7 +73,15 @@ def open_db(data_base): def extract(source, query): - """Extracts (datetime,url,title) from source database.""" + """Get records from source database. + + Args: + source: File path to the source database where we want to extract the + data from. + query: The query string to be executed in order to retrieve relevant + attributes as (datetime, url, time) from the source database according + to the browser chosen. + """ try: conn = open_db(source) cursor = conn.cursor() @@ -89,10 +97,15 @@ def extract(source, query): def clean(history): """Clean up records from source database. - Receives a list of records:(datetime,url,title). And clean all records - in place, that has a NULL/None datetime attribute. Otherwise qutebrowser - will throw errors. Also, will add a 4th attribute of '0' for the redirect - field in history.sqlite in qutebrowser.""" + + Receives a list of record and sanityze them in order for them to be + properly imported to qutebrowser. Sanitation requires addiing a 4th + attribute 'redirect' which is filled with '0's, and also purging all + records that have a NULL/None datetime attribute. + + Args: + history: List of records (datetime, url, title) from source database. + """ nulls = [record for record in history if record[0] is None] for null_datetime in nulls: history.remove(null_datetime) @@ -103,9 +116,13 @@ def clean(history): def insert_qb(history, dest): - """Insert history into dest database - Given a list of records in history and a dest db, insert all records in - the dest db.""" + """Insert history into dest database. + + Args: + history: List of records. + dest: File path to the destination database, where history will be + inserted. + """ conn = open_db(dest) cursor = conn.cursor() cursor.executemany( From 95f8c07d7fbd8e45b244e3d5d9d381fb58ea411a Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 00:31:53 +0100 Subject: [PATCH 008/322] lazy sessions --- qutebrowser/browser/qutescheme.py | 7 +++++++ qutebrowser/config/configdata.yml | 8 ++++++++ qutebrowser/misc/sessions.py | 17 ++++++++++++++++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 11dcfe004..f943fcba4 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -424,6 +424,13 @@ def qute_settings(url): confget=config.instance.get_str) return 'text/html', html +@add_handler('back') +def qute_back(url): + """Handler for qute://back. Simple page to free ram / lazy load a site, + goes back on focusing the tab.""" + + html = jinja.render('back.html', title='Suspended') + return 'text/html', html @add_handler('configdiff') def qute_configdiff(url): diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index ba8c36857..7284d98d5 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -88,6 +88,14 @@ session_default_name: If this is set to null, the session which was last loaded is saved. +session_lazy_restore: + type: + name: Bool + none_ok: true + default: false + desc: >- + Load a restored tab as soon as it takes focus. + backend: type: name: String diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 064d8c9e9..471805da9 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -347,7 +347,9 @@ class SessionManager(QObject): if 'pinned' in histentry: new_tab.data.pinned = histentry['pinned'] - active = histentry.get('active', False) + active = (histentry.get('active', False) and + (not config.val.session_lazy_restore or + histentry['url'].startswith('qute://'))) url = QUrl.fromEncoded(histentry['url'].encode('ascii')) if 'original-url' in histentry: orig_url = QUrl.fromEncoded( @@ -360,6 +362,19 @@ class SessionManager(QObject): entries.append(entry) if active: new_tab.title_changed.emit(histentry['title']) + + if config.val.session_lazy_restore and data['history']: + last = data['history'][-1] + title = last['title'] + url = 'qute://back#' + title + active = last.get('active', False) + + if not last['url'].startswith('qute://'): + entries.append(TabHistoryItem(url=QUrl.fromEncoded(url.encode('ascii')), + title=title, active=active, user_data={})) + if active: + new_tab.title_changed.emit(title) + try: new_tab.history.load_items(entries) except ValueError as e: From ade7004f8f4429affc27a8f5b03ad783ca994ca9 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 00:48:31 +0100 Subject: [PATCH 009/322] lazy sessions --- qutebrowser/browser/qutescheme.py | 2 ++ qutebrowser/misc/sessions.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index f943fcba4..3881a0d2e 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -424,6 +424,7 @@ def qute_settings(url): confget=config.instance.get_str) return 'text/html', html + @add_handler('back') def qute_back(url): """Handler for qute://back. Simple page to free ram / lazy load a site, @@ -432,6 +433,7 @@ def qute_back(url): html = jinja.render('back.html', title='Suspended') return 'text/html', html + @add_handler('configdiff') def qute_configdiff(url): """Handler for qute://configdiff.""" diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 471805da9..4784f796a 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -348,8 +348,8 @@ class SessionManager(QObject): new_tab.data.pinned = histentry['pinned'] active = (histentry.get('active', False) and - (not config.val.session_lazy_restore or - histentry['url'].startswith('qute://'))) + (not config.val.session_lazy_restore or + histentry['url'].startswith('qute://'))) url = QUrl.fromEncoded(histentry['url'].encode('ascii')) if 'original-url' in histentry: orig_url = QUrl.fromEncoded( @@ -370,8 +370,10 @@ class SessionManager(QObject): active = last.get('active', False) if not last['url'].startswith('qute://'): - entries.append(TabHistoryItem(url=QUrl.fromEncoded(url.encode('ascii')), - title=title, active=active, user_data={})) + entries.append(TabHistoryItem( + url=QUrl.fromEncoded(url.encode('ascii')), + title=title, active=active, user_data={})) + if active: new_tab.title_changed.emit(title) From 51dea053f459c7cd0977c75047ff41a2f3ca1060 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 01:00:16 +0100 Subject: [PATCH 010/322] lazy sessions --- qutebrowser/html/back.html | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 qutebrowser/html/back.html diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html new file mode 100644 index 000000000..8a21f3d80 --- /dev/null +++ b/qutebrowser/html/back.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} + +{% block script %} +window.onload = function() { + var title = 'Suspended: ' + document.location.hash.substr(1); + var node = document.getElementsByTagName('h1')[0]; + node.innerText = document.title = title; +}; +window.onfocus = function() { + window.history.back(); +}; +{% endblock %} + +{% block content %} + +

{{ title }}

+ +{% endblock %} From c4bb13431331b7c5aad5696f54998210c12a1f12 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 11:04:04 +0100 Subject: [PATCH 011/322] lazy sessions, improved version --- qutebrowser/misc/sessions.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 4784f796a..da49205df 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -323,6 +323,18 @@ class SessionManager(QObject): def _load_tab(self, new_tab, data): """Load yaml data into a newly opened tab.""" entries = [] + + if config.val.session_lazy_restore and data['history']: + last = data['history'][-1] + + if not last['url'].startswith('qute://'): + data['history'].append({ + 'title': last['title'], + 'url': 'qute://back#' + last['title'], + 'active': last.get('active', False) + }) + last['active'] = False + for histentry in data['history']: user_data = {} @@ -347,9 +359,7 @@ class SessionManager(QObject): if 'pinned' in histentry: new_tab.data.pinned = histentry['pinned'] - active = (histentry.get('active', False) and - (not config.val.session_lazy_restore or - histentry['url'].startswith('qute://'))) + active = histentry.get('active', False) url = QUrl.fromEncoded(histentry['url'].encode('ascii')) if 'original-url' in histentry: orig_url = QUrl.fromEncoded( @@ -363,20 +373,6 @@ class SessionManager(QObject): if active: new_tab.title_changed.emit(histentry['title']) - if config.val.session_lazy_restore and data['history']: - last = data['history'][-1] - title = last['title'] - url = 'qute://back#' + title - active = last.get('active', False) - - if not last['url'].startswith('qute://'): - entries.append(TabHistoryItem( - url=QUrl.fromEncoded(url.encode('ascii')), - title=title, active=active, user_data={})) - - if active: - new_tab.title_changed.emit(title) - try: new_tab.history.load_items(entries) except ValueError as e: From c150c5481a093a14c42ef1233cc54214972bc381 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 13:46:50 +0100 Subject: [PATCH 012/322] lazy sessions, dont save qute://back --- qutebrowser/misc/sessions.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index da49205df..807d4bde4 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -205,7 +205,11 @@ class SessionManager(QObject): for idx, item in enumerate(tab.history): qtutils.ensure_valid(item) item_data = self._save_tab_item(tab, idx, item) - data['history'].append(item_data) + if item_data['url'].startswith('qute://back'): + if 'active' in item_data and data['history']: + data['history'][-1]['active'] = item_data.get('active', False) + else: + data['history'].append(item_data) return data def _save_all(self, *, only_window=None, with_private=False): From 1a33c88c9607fdfd71b925699994981ce19a8827 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 13:47:57 +0100 Subject: [PATCH 013/322] lazy sessions, dont save qute://back --- qutebrowser/misc/sessions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 807d4bde4..1dc27397d 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -207,7 +207,8 @@ class SessionManager(QObject): item_data = self._save_tab_item(tab, idx, item) if item_data['url'].startswith('qute://back'): if 'active' in item_data and data['history']: - data['history'][-1]['active'] = item_data.get('active', False) + data['history'][-1]['active'] = \ + item_data.get('active', False) else: data['history'].append(item_data) return data From 2debeafe1bd5ee7dfb0a88a57957554fd03934d2 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 13:51:30 +0100 Subject: [PATCH 014/322] lazy sessions, dont save qute://back --- qutebrowser/misc/sessions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 1dc27397d..c44b2ed8b 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -206,9 +206,8 @@ class SessionManager(QObject): qtutils.ensure_valid(item) item_data = self._save_tab_item(tab, idx, item) if item_data['url'].startswith('qute://back'): - if 'active' in item_data and data['history']: - data['history'][-1]['active'] = \ - item_data.get('active', False) + if item_data.get('active', False) and data['history']: + data['history'][-1]['active'] = True else: data['history'].append(item_data) return data From cf8130bd225d4b4497f79d3ccd1d9a3226b7a6b8 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 14:12:37 +0100 Subject: [PATCH 015/322] lazy session, fix: active entry is not the end of the history --- qutebrowser/html/back.html | 14 +++++++++++--- qutebrowser/misc/sessions.py | 26 +++++++++++++++++--------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index 8a21f3d80..c28419a64 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -6,9 +6,17 @@ window.onload = function() { var node = document.getElementsByTagName('h1')[0]; node.innerText = document.title = title; }; -window.onfocus = function() { - window.history.back(); -}; +setTimeout(function() { + /* drop first focus event, to avoid problems + (allow to go easily to newer history entries) */ + var triggered = false; + window.onfocus = function() { + if (! triggered) { + triggered = true; + window.history.back(); + } + }; +}, 1000); {% endblock %} {% block content %} diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index c44b2ed8b..0e2b40c68 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -206,7 +206,9 @@ class SessionManager(QObject): qtutils.ensure_valid(item) item_data = self._save_tab_item(tab, idx, item) if item_data['url'].startswith('qute://back'): + # dont add qute://back to the session file if item_data.get('active', False) and data['history']: + # mark entry before qute://back as active data['history'][-1]['active'] = True else: data['history'].append(item_data) @@ -331,15 +333,7 @@ class SessionManager(QObject): if config.val.session_lazy_restore and data['history']: last = data['history'][-1] - if not last['url'].startswith('qute://'): - data['history'].append({ - 'title': last['title'], - 'url': 'qute://back#' + last['title'], - 'active': last.get('active', False) - }) - last['active'] = False - - for histentry in data['history']: + for i, histentry in enumerate(data['history']): user_data = {} if 'zoom' in data: @@ -363,6 +357,20 @@ class SessionManager(QObject): if 'pinned' in histentry: new_tab.data.pinned = histentry['pinned'] + if (config.val.session_lazy_restore and + histentry.get('active', False) and + not histentry['url'].startswith('qute://back')): + # remove "active" mark and insert back page marked as active + data['history'].insert( + i + 1, + { + 'title': histentry['title'], + 'url': 'qute://back#' + histentry['title'], + 'active': True + }) + histentry['active'] = False + + print(histentry) active = histentry.get('active', False) url = QUrl.fromEncoded(histentry['url'].encode('ascii')) if 'original-url' in histentry: From 13dc24f6cafe297b570b60d1ca28ae4f6a76d3c4 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Sat, 18 Nov 2017 14:31:55 +0100 Subject: [PATCH 016/322] debug code removed --- qutebrowser/misc/sessions.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 0e2b40c68..c3634a64a 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -330,9 +330,6 @@ class SessionManager(QObject): """Load yaml data into a newly opened tab.""" entries = [] - if config.val.session_lazy_restore and data['history']: - last = data['history'][-1] - for i, histentry in enumerate(data['history']): user_data = {} @@ -370,7 +367,6 @@ class SessionManager(QObject): }) histentry['active'] = False - print(histentry) active = histentry.get('active', False) url = QUrl.fromEncoded(histentry['url'].encode('ascii')) if 'original-url' in histentry: From aa40842848a60039c3978e0e7b8d0c3c4eb8f6e5 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Tue, 21 Nov 2017 00:38:51 +0100 Subject: [PATCH 017/322] lazy sessions, docstring formatted, settings renamed, javascript notice changed, insert method changed --- doc/help/commands.asciidoc | 2 +- doc/help/settings.asciidoc | 6 +++--- qutebrowser/browser/qutescheme.py | 8 +++++--- qutebrowser/config/configdata.yml | 8 +++++--- qutebrowser/html/back.html | 16 +++------------- qutebrowser/misc/sessions.py | 27 ++++++++++++++++++--------- tests/unit/misc/test_sessions.py | 2 +- 7 files changed, 36 insertions(+), 33 deletions(-) diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 04377c055..aa94de305 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -1070,7 +1070,7 @@ Syntax: +:session-save [*--current*] [*--quiet*] [*--force*] [*--only-active-win Save a session. ==== positional arguments -* +'name'+: The name of the session. If not given, the session configured in session_default_name is saved. +* +'name'+: The name of the session. If not given, the session configured in session.default_name is saved. ==== optional arguments diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 64881f619..9f694c184 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -222,7 +222,7 @@ |<>|Turn on Qt HighDPI scaling. |<>|Show a scrollbar. |<>|Enable smooth scrolling for web pages. -|<>|Name of the session to save by default. +|<>|Name of the session to save by default. |<>|Languages to use for spell checking. |<>|Hide the statusbar unless a message is shown. |<>|Padding (in pixels) for the statusbar. @@ -2554,8 +2554,8 @@ Type: <> Default: +pass:[false]+ -[[session_default_name]] -=== session_default_name +[[session.default_name]] +=== session.default_name Name of the session to save by default. If this is set to null, the session which was last loaded is saved. diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 3881a0d2e..4db7bdc84 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -427,10 +427,12 @@ def qute_settings(url): @add_handler('back') def qute_back(url): - """Handler for qute://back. Simple page to free ram / lazy load a site, - goes back on focusing the tab.""" + """Handler for qute://back. - html = jinja.render('back.html', title='Suspended') + Simple page to free ram / lazy load a site, goes back on focusing the tab. + """ + html = jinja.render('back.html', + title='Suspended: ' + url.url().split('#')[-1]) return 'text/html', html diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 7284d98d5..7f362daa4 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -79,6 +79,9 @@ new_instance_open_target_window: When `new_instance_open_target` is not set to `window`, this is ignored. session_default_name: + renamed: session.default_name + +session.default_name: type: name: SessionName none_ok: true @@ -88,13 +91,12 @@ session_default_name: If this is set to null, the session which was last loaded is saved. -session_lazy_restore: +session.lazy_restore: type: name: Bool none_ok: true default: false - desc: >- - Load a restored tab as soon as it takes focus. + desc: Load a restored tab as soon as it takes focus. backend: type: diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index c28419a64..0d06158f2 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -1,26 +1,16 @@ {% extends "base.html" %} {% block script %} -window.onload = function() { - var title = 'Suspended: ' + document.location.hash.substr(1); - var node = document.getElementsByTagName('h1')[0]; - node.innerText = document.title = title; -}; setTimeout(function() { /* drop first focus event, to avoid problems (allow to go easily to newer history entries) */ - var triggered = false; window.onfocus = function() { - if (! triggered) { - triggered = true; - window.history.back(); - } + window.onfocus = null; + window.history.back(); }; }, 1000); {% endblock %} {% block content %} - -

{{ title }}

- + {% endblock %} diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index c3634a64a..faee99c99 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -22,6 +22,8 @@ import os import os.path +from itertools import chain, dropwhile, takewhile + import sip from PyQt5.QtCore import QUrl, QObject, QPoint, QTimer from PyQt5.QtWidgets import QApplication @@ -205,8 +207,8 @@ class SessionManager(QObject): for idx, item in enumerate(tab.history): qtutils.ensure_valid(item) item_data = self._save_tab_item(tab, idx, item) - if item_data['url'].startswith('qute://back'): - # dont add qute://back to the session file + if item.url().url().startswith('qute://back'): + # don't add qute://back to the session file if item_data.get('active', False) and data['history']: # mark entry before qute://back as active data['history'][-1]['active'] = True @@ -257,7 +259,7 @@ class SessionManager(QObject): object. """ if name is default: - name = config.val.session_default_name + name = config.val.session.default_name if name is None: if self._current is not None: name = self._current @@ -329,8 +331,16 @@ class SessionManager(QObject): def _load_tab(self, new_tab, data): """Load yaml data into a newly opened tab.""" entries = [] + lazy_load = [] + # use len(data['history']) + # -> dropwhile empty if not session.lazy_session + lazy_index = len(data['history']) + gen = chain( + takewhile(lambda _: not lazy_load, enumerate(data['history'])), + enumerate(lazy_load), + dropwhile(lambda i: i[0] < lazy_index, enumerate(data['history']))) - for i, histentry in enumerate(data['history']): + for i, histentry in gen: user_data = {} if 'zoom' in data: @@ -354,13 +364,12 @@ class SessionManager(QObject): if 'pinned' in histentry: new_tab.data.pinned = histentry['pinned'] - if (config.val.session_lazy_restore and + if (config.val.session.lazy_restore and histentry.get('active', False) and not histentry['url'].startswith('qute://back')): # remove "active" mark and insert back page marked as active - data['history'].insert( - i + 1, - { + lazy_index = i + 1 + lazy_load.append({ 'title': histentry['title'], 'url': 'qute://back#' + histentry['title'], 'active': True @@ -481,7 +490,7 @@ class SessionManager(QObject): Args: name: The name of the session. If not given, the session configured - in session_default_name is saved. + in session.default_name is saved. current: Save the current session instead of the default. quiet: Don't show confirmation message. force: Force saving internal sessions (starting with an underline). diff --git a/tests/unit/misc/test_sessions.py b/tests/unit/misc/test_sessions.py index 771430d5b..b2cb8a3dd 100644 --- a/tests/unit/misc/test_sessions.py +++ b/tests/unit/misc/test_sessions.py @@ -170,7 +170,7 @@ class TestSaveAll: ]) def test_get_session_name(config_stub, sess_man, arg, config, current, expected): - config_stub.val.session_default_name = config + config_stub.val.session.default_name = config sess_man._current = current assert sess_man._get_session_name(arg) == expected From 607cd9ba6ee568849b111c2a950acb5c3ec0d9e8 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Tue, 21 Nov 2017 01:19:04 +0100 Subject: [PATCH 018/322] indent adjusted --- qutebrowser/misc/sessions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index faee99c99..7a26d161c 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -370,10 +370,10 @@ class SessionManager(QObject): # remove "active" mark and insert back page marked as active lazy_index = i + 1 lazy_load.append({ - 'title': histentry['title'], - 'url': 'qute://back#' + histentry['title'], - 'active': True - }) + 'title': histentry['title'], + 'url': 'qute://back#' + histentry['title'], + 'active': True + }) histentry['active'] = False active = histentry.get('active', False) From e2d5a443cc746885f6fb3512470298b168904e29 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Tue, 21 Nov 2017 23:57:06 +0100 Subject: [PATCH 019/322] lazy sessions --- qutebrowser/browser/qutescheme.py | 2 +- qutebrowser/config/configdata.yml | 4 +--- qutebrowser/html/back.html | 2 +- qutebrowser/misc/sessions.py | 15 ++++++++------- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 4db7bdc84..f12ef529a 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -432,7 +432,7 @@ def qute_back(url): Simple page to free ram / lazy load a site, goes back on focusing the tab. """ html = jinja.render('back.html', - title='Suspended: ' + url.url().split('#')[-1]) + title='Suspended: ' + url.fragment()) return 'text/html', html diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 7f362daa4..39c0fa6b1 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -92,9 +92,7 @@ session.default_name: If this is set to null, the session which was last loaded is saved. session.lazy_restore: - type: - name: Bool - none_ok: true + type: Bool default: false desc: Load a restored tab as soon as it takes focus. diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index 0d06158f2..83f5cb74e 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -12,5 +12,5 @@ setTimeout(function() { {% endblock %} {% block content %} - + {% endblock %} diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 7a26d161c..64d170dab 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -21,8 +21,7 @@ import os import os.path - -from itertools import chain, dropwhile, takewhile +import itertools import sip from PyQt5.QtCore import QUrl, QObject, QPoint, QTimer @@ -207,7 +206,7 @@ class SessionManager(QObject): for idx, item in enumerate(tab.history): qtutils.ensure_valid(item) item_data = self._save_tab_item(tab, idx, item) - if item.url().url().startswith('qute://back'): + if item.url().scheme() == 'qute' and item.url().host() == 'back': # don't add qute://back to the session file if item_data.get('active', False) and data['history']: # mark entry before qute://back as active @@ -335,10 +334,12 @@ class SessionManager(QObject): # use len(data['history']) # -> dropwhile empty if not session.lazy_session lazy_index = len(data['history']) - gen = chain( - takewhile(lambda _: not lazy_load, enumerate(data['history'])), - enumerate(lazy_load), - dropwhile(lambda i: i[0] < lazy_index, enumerate(data['history']))) + gen = itertools.chain( + itertools.takewhile(lambda _: not lazy_load, + enumerate(data['history'])), + enumerate(lazy_load), + itertools.dropwhile(lambda i: i[0] < lazy_index, + enumerate(data['history']))) for i, histentry in gen: user_data = {} From 9df149fe8fc0822fcdc6a718e3542782aca22fe6 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Fri, 24 Nov 2017 17:15:26 +0100 Subject: [PATCH 020/322] urlencode fix --- qutebrowser/browser/qutescheme.py | 6 ++++-- qutebrowser/misc/sessions.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index f12ef529a..9771f6db1 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -29,6 +29,7 @@ import os import time import textwrap import mimetypes +import urllib import pkg_resources from PyQt5.QtCore import QUrlQuery, QUrl @@ -431,8 +432,9 @@ def qute_back(url): Simple page to free ram / lazy load a site, goes back on focusing the tab. """ - html = jinja.render('back.html', - title='Suspended: ' + url.fragment()) + html = jinja.render( + 'back.html', + title='Suspended: ' + urllib.parse.unquote(url.fragment())) return 'text/html', html diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 64d170dab..1b0106f1b 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -22,6 +22,7 @@ import os import os.path import itertools +import urllib import sip from PyQt5.QtCore import QUrl, QObject, QPoint, QTimer @@ -372,7 +373,9 @@ class SessionManager(QObject): lazy_index = i + 1 lazy_load.append({ 'title': histentry['title'], - 'url': 'qute://back#' + histentry['title'], + 'url': + 'qute://back#' + + urllib.parse.quote(histentry['title']), 'active': True }) histentry['active'] = False From 568d60753e4856e9f84921e1084ea23d4e9e5956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andor=20Uhl=C3=A1r?= Date: Sun, 10 May 2015 18:33:36 +0200 Subject: [PATCH 021/322] Add greasemonkey compatible userscript module. WebKit backend only for now. Loads all .js files from a directory, specified in the greasemonkey-directory key in the storage section, defaulting to data/greasemonkey, and wraps them in a minimal environment providing some GM_* functions. Makes those scripts available via the "greasemonkey" registered object in objreg and injects scripts at appropriate times in a page load base on @run-at directives. --- qutebrowser/browser/browsertab.py | 6 +- qutebrowser/browser/greasemonkey.py | 262 ++++++++++++++++++++++++++ qutebrowser/browser/webkit/webpage.py | 28 +++ qutebrowser/utils/log.py | 4 +- 4 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 qutebrowser/browser/greasemonkey.py diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index 547e276db..866943f87 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -31,7 +31,7 @@ from qutebrowser.keyinput import modeman from qutebrowser.config import config from qutebrowser.utils import utils, objreg, usertypes, log, qtutils from qutebrowser.misc import miscwidgets, objects -from qutebrowser.browser import mouse, hints +from qutebrowser.browser import mouse, hints, greasemonkey tab_id_gen = itertools.count(0) @@ -64,6 +64,10 @@ def init(): from qutebrowser.browser.webengine import webenginetab webenginetab.init() + log.init.debug("Initializing Greasemonkey...") + gm_manager = greasemonkey.GreasemonkeyManager() + objreg.register('greasemonkey', gm_manager) + class WebTabError(Exception): diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py new file mode 100644 index 000000000..d1f63a4a4 --- /dev/null +++ b/qutebrowser/browser/greasemonkey.py @@ -0,0 +1,262 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2017 Florian Bruhin (The Compiler) +# +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Load, parse and make avalaible greasemonkey scripts.""" + +import re +import os +import json +import fnmatch +import functools +import glob + +from PyQt5.QtCore import pyqtSignal, QObject + +from qutebrowser.utils import log, standarddir + +# TODO: GM_ bootstrap + + +def _scripts_dir(): + """Get the directory of the scripts.""" + return os.path.join(standarddir.data(), 'greasemonkey') + + +class GreasemonkeyScript: + """Container class for userscripts, parses metadata blocks.""" + + GM_BOOTSTRAP_TEMPLATE = r"""var _qute_script_id = "__gm_{scriptName}"; + +function GM_log(text) {{ + console.log(text); +}} + +GM_info = (function() {{ + return {{ + 'script': {scriptInfo}, + 'scriptMetaStr': {scriptMeta}, + 'scriptWillUpdate': false, + 'version': '0.0.1', + 'scriptHandler': 'Tampermonkey' //so scripts don't expect exportFunction + }}; +}}()); + +function GM_setValue(key, value) {{ + if (localStorage !== null && + typeof key === "string" && + (typeof value === "string" || + typeof value === "number" || + typeof value == "boolean")) {{ + localStorage.setItem(_qute_script_id + key, value); + }} +}} + +function GM_getValue(key, default_) {{ + if (localStorage !== null && typeof key === "string") {{ + return localStorage.getItem(_qute_script_id + key) || default_; + }} +}} + +function GM_deleteValue(key) {{ + if (localStorage !== null && typeof key === "string") {{ + localStorage.removeItem(_qute_script_id + key); + }} +}} + +function GM_listValues() {{ + var i; + var keys = []; + for (i = 0; i < localStorage.length; ++i) {{ + if (localStorage.key(i).startsWith(_qute_script_id)) {{ + keys.push(localStorage.key(i)); + }} + }} + return keys; +}} + +function GM_openInTab(url) {{ + window.open(url); +}} + + +// Almost verbatim copy from Eric +function GM_xmlhttpRequest(/* object */ details) {{ + details.method = details.method.toUpperCase() || "GET"; + + if(!details.url) {{ + throw("GM_xmlhttpRequest requires an URL."); + }} + + // build XMLHttpRequest object + var oXhr = new XMLHttpRequest; + // run it + if("onreadystatechange" in details) + oXhr.onreadystatechange = function() {{ + details.onreadystatechange(oXhr) + }}; + if("onload" in details) + oXhr.onload = function() {{ details.onload(oXhr) }}; + if("onerror" in details) + oXhr.onerror = function() {{ details.onerror(oXhr) }}; + + oXhr.open(details.method, details.url, true); + + if("headers" in details) + for(var header in details.headers) + oXhr.setRequestHeader(header, details.headers[header]); + + if("data" in details) + oXhr.send(details.data); + else + oXhr.send(); +}} + +function GM_addStyle(/* String */ styles) {{ + var head = document.getElementsByTagName("head")[0]; + if (head === undefined) {{ + document.onreadystatechange = function() {{ + if (document.readyState == "interactive") {{ + var oStyle = document.createElement("style"); + oStyle.setAttribute("type", "text/css"); + oStyle.appendChild(document.createTextNode(styles)); + document.getElementsByTagName("head")[0].appendChild(oStyle); + }} + }} + }} + else {{ + var oStyle = document.createElement("style"); + oStyle.setAttribute("type", "text/css"); + oStyle.appendChild(document.createTextNode(styles)); + head.appendChild(oStyle); + }} +}} + +unsafeWindow = window; +""" + + def __init__(self, properties, code): + self._code = code + self.includes = [] + self.excludes = [] + self.description = None + self.name = None + self.run_at = None + for name, value in properties: + if name == 'name': + self.name = value + elif name == 'description': + self.description = value + elif name in ['include', 'match']: + self.includes.append(value) + elif name in ['exclude', 'exclude_match']: + self.excludes.append(value) + elif name == 'run-at': + self.run_at = value + + HEADER_REGEX = r'// ==UserScript==.|\n+// ==/UserScript==\n' + PROPS_REGEX = r'// @(?P[^\s]+)\s+(?P.+)' + + @classmethod + def parse(cls, source): + """GreaseMonkeyScript factory. + + Takes a userscript source and returns a GreaseMonkeyScript. + Parses the greasemonkey metadata block, if present, to fill out + attributes. + """ + matches = re.split(cls.HEADER_REGEX, source, maxsplit=1) + try: + props, _code = matches + except ValueError: + props = "" + script = cls(re.findall(cls.PROPS_REGEX, props), code) + script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:])) + return script + + def code(self): + """Return the processed javascript code of this script. + + Adorns the source code with GM_* methods for greasemonkey + compatibility and wraps it in an IFFE to hide it within a + lexical scope. Note that this means line numbers in your + browser's debugger/inspector will not match up to the line + numbers in the source script directly. + """ + gm_bootstrap = self.GM_BOOTSTRAP_TEMPLATE.format( + scriptName=self.name, + scriptInfo=self._meta_json(), + scriptMeta=self.script_meta) + return '\n'.join([gm_bootstrap, self._code]) + + def _meta_json(self): + return json.dumps({ + 'name': self.name, + 'description': self.description, + 'matches': self.includes, + 'includes': self.includes, + 'excludes': self.excludes, + 'run-at': self.run_at, + }) + + +class GreasemonkeyManager: + + def __init__(self, parent=None): + super().__init__(parent) + self._run_start = [] + self._run_end = [] + + scripts_dir = os.path.abspath(_scripts_dir()) + log.greasemonkey.debug("Reading scripts from: {}".format(scripts_dir)) + for script_filename in glob.glob(os.path.join(scripts_dir, '*.js')): + if not os.path.isfile(script_filename): + continue + script_path = os.path.join(scripts_dir, script_filename) + with open(script_path, encoding='utf-8') as script_file: + script = GreasemonkeyScript.parse(script_file.read()) + if not script.name: + script.name = script_filename + + if script.run_at == 'document-start': + self._run_start.append(script) + elif script.run_at == 'document-end': + self._run_end.append(script) + else: + log.greasemonkey.warning("Script {} has invalid run-at " + "defined, ignoring." + .format(script_path)) + continue + log.greasemonkey.debug("Loaded script: {}".format(script.name)) + + def scripts_for(self, url): + """Fetch scripts that are registered to run for url. + + returns a tuple of lists of scripts meant to run at (document-start, + document-end) + """ + match = functools.partial(fnmatch.fnmatch, url) + tester = (lambda script: + any(map(match, script.includes())) and not + any(map(match, script.excludes()))) + return (list(filter(tester, self._run_start)), + list(filter(tester, self._run_end))) + + def all_scripts(self): + """Return all scripts found in the configured script directory.""" + return self._run_start + self._run_end diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 7e1d991b9..9beba6ddc 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -86,6 +86,10 @@ class BrowserPage(QWebPage): self.on_save_frame_state_requested) self.restoreFrameStateRequested.connect( self.on_restore_frame_state_requested) + self.mainFrame().javaScriptWindowObjectCleared.connect( + functools.partial(self.inject_userjs, load='start')) + self.mainFrame().initialLayoutCompleted.connect( + functools.partial(self.inject_userjs, load='end')) def javaScriptPrompt(self, frame, js_msg, default): """Override javaScriptPrompt to use qutebrowser prompts.""" @@ -283,6 +287,30 @@ class BrowserPage(QWebPage): else: self.error_occurred = False + @pyqtSlot() + def inject_userjs(self, load): + """Inject user javascripts into the page. + + param: The page load stage to inject the corresponding scripts + for. Support values are "start" and "end", + corresponding to the allowed values of the `@run-at` + directive in the greasemonkey metadata spec. + """ + greasemonkey = objreg.get('greasemonkey') + url = self.currentFrame().url() + start_scripts, end_scripts = greasemonkey.scripts_for(url.toDisplayString()) + log.greasemonkey.debug('scripts: {}'.format(start_scripts if start else end_scripts)) + + toload = [] + if load == "start": + toload = start_scripts + elif load == "end": + toload = end_scripts + + for script in toload: + log.webview.debug('Running GM script: {}'.format(script.name)) + self.currentFrame().evaluateJavaScript(script.code()) + @pyqtSlot('QWebFrame*', 'QWebPage::Feature') def _on_feature_permission_requested(self, frame, feature): """Ask the user for approval for geolocation/notifications.""" diff --git a/qutebrowser/utils/log.py b/qutebrowser/utils/log.py index 68cf1d2ba..dc0ff5580 100644 --- a/qutebrowser/utils/log.py +++ b/qutebrowser/utils/log.py @@ -95,7 +95,8 @@ LOGGER_NAMES = [ 'commands', 'signals', 'downloads', 'js', 'qt', 'rfc6266', 'ipc', 'shlexer', 'save', 'message', 'config', 'sessions', - 'webelem', 'prompt', 'network', 'sql' + 'webelem', 'prompt', 'network', 'sql', + 'greasemonkey' ] @@ -144,6 +145,7 @@ webelem = logging.getLogger('webelem') prompt = logging.getLogger('prompt') network = logging.getLogger('network') sql = logging.getLogger('sql') +greasemonkey = logging.getLogger('greasemonkey') ram_handler = None From ecdde7663ffd44579c5935c5be78d92bc9816885 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 7 Jun 2017 15:40:14 +1200 Subject: [PATCH 022/322] Add greasemonkey-reload command. Also add a signal to emit when scripts are reloaded. Had to make GreasemonkeyManager inherit from QObject to get signals to work. --- qutebrowser/browser/greasemonkey.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index d1f63a4a4..d01a8f8e5 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -29,6 +29,7 @@ import glob from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.utils import log, standarddir +from qutebrowser.commands import cmdutils # TODO: GM_ bootstrap @@ -215,10 +216,26 @@ unsafeWindow = window; }) -class GreasemonkeyManager: +class GreasemonkeyManager(QObject): + + """Manager of userscripts and a greasemonkey compatible environment. + + Signals: + scripts_reloaded: Emitted when scripts are reloaded from disk. + Any any cached or already-injected scripts should be + considered obselete. + """ + + scripts_reloaded = pyqtSignal() def __init__(self, parent=None): super().__init__(parent) + self.load_scripts() + + @cmdutils.register(name='greasemonkey-reload', + instance='greasemonkey') + def load_scripts(self): + """Re-Read greasemonkey scripts from disk.""" self._run_start = [] self._run_end = [] @@ -243,6 +260,7 @@ class GreasemonkeyManager: .format(script_path)) continue log.greasemonkey.debug("Loaded script: {}".format(script.name)) + self.scripts_reloaded.emit() def scripts_for(self, url): """Fetch scripts that are registered to run for url. From 13728387d7d875bbe7bed5687330479e2e4bdc66 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 7 Jun 2017 15:41:53 +1200 Subject: [PATCH 023/322] Greasemonkey: Fix crash on undefined metadata. --- qutebrowser/browser/greasemonkey.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index d01a8f8e5..ef8635178 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -256,9 +256,12 @@ class GreasemonkeyManager(QObject): self._run_end.append(script) else: log.greasemonkey.warning("Script {} has invalid run-at " - "defined, ignoring." + "defined, defaulting to " + "document-end" .format(script_path)) - continue + # Default as per + # https://wiki.greasespot.net/Metadata_Block#.40run-at + self._run_end.append(script) log.greasemonkey.debug("Loaded script: {}".format(script.name)) self.scripts_reloaded.emit() From 25f626a436d1f898391e19fb531984d28bb0547f Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 7 Jun 2017 15:45:12 +1200 Subject: [PATCH 024/322] Greasemonkey: Add run-at document-idle. Supposed to be after all the assets have finished loading and in page js has run. Not that we can garuntee that last bit. If a script misbehaves because a precondition isn't yet met I suggest adding a defer method to the script that adds a timer until the precondition is met. Also changed the map/filter calls to use list comprehensions to keep pylint happy. Even if it does look uglier. --- qutebrowser/browser/greasemonkey.py | 18 ++++++++++++------ qutebrowser/browser/webkit/webpage.py | 12 +++++++----- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index ef8635178..058a2d0f4 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -238,6 +238,7 @@ class GreasemonkeyManager(QObject): """Re-Read greasemonkey scripts from disk.""" self._run_start = [] self._run_end = [] + self._run_idle = [] scripts_dir = os.path.abspath(_scripts_dir()) log.greasemonkey.debug("Reading scripts from: {}".format(scripts_dir)) @@ -254,6 +255,8 @@ class GreasemonkeyManager(QObject): self._run_start.append(script) elif script.run_at == 'document-end': self._run_end.append(script) + elif script.run_at == 'document-idle': + self._run_idle.append(script) else: log.greasemonkey.warning("Script {} has invalid run-at " "defined, defaulting to " @@ -269,15 +272,18 @@ class GreasemonkeyManager(QObject): """Fetch scripts that are registered to run for url. returns a tuple of lists of scripts meant to run at (document-start, - document-end) + document-end, document-idle) """ match = functools.partial(fnmatch.fnmatch, url) tester = (lambda script: - any(map(match, script.includes())) and not - any(map(match, script.excludes()))) - return (list(filter(tester, self._run_start)), - list(filter(tester, self._run_end))) + any([match(pat) for pat in script.includes]) and + not any([match(pat) for pat in script.excludes])) + return ( + [script for script in self._run_start if tester(script)], + [script for script in self._run_end if tester(script)], + [script for script in self._run_idle if tester(script)] + ) def all_scripts(self): """Return all scripts found in the configured script directory.""" - return self._run_start + self._run_end + return self._run_start + self._run_end + self._run_idle diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 9beba6ddc..095e41fe2 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -90,6 +90,8 @@ class BrowserPage(QWebPage): functools.partial(self.inject_userjs, load='start')) self.mainFrame().initialLayoutCompleted.connect( functools.partial(self.inject_userjs, load='end')) + self.mainFrame().loadFinished.connect( + functools.partial(self.inject_userjs, load='idle')) def javaScriptPrompt(self, frame, js_msg, default): """Override javaScriptPrompt to use qutebrowser prompts.""" @@ -292,20 +294,20 @@ class BrowserPage(QWebPage): """Inject user javascripts into the page. param: The page load stage to inject the corresponding scripts - for. Support values are "start" and "end", + for. Support values are "start", "end" and "idle", corresponding to the allowed values of the `@run-at` directive in the greasemonkey metadata spec. """ greasemonkey = objreg.get('greasemonkey') url = self.currentFrame().url() - start_scripts, end_scripts = greasemonkey.scripts_for(url.toDisplayString()) - log.greasemonkey.debug('scripts: {}'.format(start_scripts if start else end_scripts)) - - toload = [] + start_scripts, end_scripts, idle_scripts = \ + greasemonkey.scripts_for(url.toDisplayString()) if load == "start": toload = start_scripts elif load == "end": toload = end_scripts + elif load == "idle": + toload = idle_scripts for script in toload: log.webview.debug('Running GM script: {}'.format(script.name)) From be9f8bd0de2d8189852bb306ebd25c010bc15c76 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 7 Jun 2017 15:49:41 +1200 Subject: [PATCH 025/322] Greasemonkey: Lift greasemonkey init app.py To prepare for multiple-backend support. --- qutebrowser/app.py | 6 +++++- qutebrowser/browser/browsertab.py | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index 2ed579f61..c32a208ac 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -64,7 +64,7 @@ from qutebrowser.completion.models import miscmodels from qutebrowser.commands import cmdutils, runners, cmdexc from qutebrowser.config import config, websettings, configfiles, configinit from qutebrowser.browser import (urlmarks, adblock, history, browsertab, - downloads) + downloads, greasemonkey) from qutebrowser.browser.network import proxy from qutebrowser.browser.webkit import cookies, cache from qutebrowser.browser.webkit.network import networkmanager @@ -491,6 +491,10 @@ def _init_modules(args, crash_handler): diskcache = cache.DiskCache(standarddir.cache(), parent=qApp) objreg.register('cache', diskcache) + log.init.debug("Initializing Greasemonkey...") + gm_manager = greasemonkey.GreasemonkeyManager() + objreg.register('greasemonkey', gm_manager) + log.init.debug("Misc initialization...") macros.init() # Init backend-specific stuff diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py index 866943f87..547e276db 100644 --- a/qutebrowser/browser/browsertab.py +++ b/qutebrowser/browser/browsertab.py @@ -31,7 +31,7 @@ from qutebrowser.keyinput import modeman from qutebrowser.config import config from qutebrowser.utils import utils, objreg, usertypes, log, qtutils from qutebrowser.misc import miscwidgets, objects -from qutebrowser.browser import mouse, hints, greasemonkey +from qutebrowser.browser import mouse, hints tab_id_gen = itertools.count(0) @@ -64,10 +64,6 @@ def init(): from qutebrowser.browser.webengine import webenginetab webenginetab.init() - log.init.debug("Initializing Greasemonkey...") - gm_manager = greasemonkey.GreasemonkeyManager() - objreg.register('greasemonkey', gm_manager) - class WebTabError(Exception): From f26377351c0fc8b2e50b583ec7eb8b195d76501c Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 7 Jun 2017 16:06:50 +1200 Subject: [PATCH 026/322] Greasemonkey: Add greasemonkey hooks for webengine. For qtwebengine 5.8+ only. This is because as of 5.8 some greasemonkey script support is in upstream. That is, qtwebenginescript(collection) parses the greasemonkey metadata block and uses @include/match/exclude to decide what sites to inject a script onto and @run-at to decide when to inject it, which saves us the trouble. Notes on doing this in <5.8 are below. Scripts are currently injected into the main "world", that is the same world as the javascript from the page. This is good because it means userscripts can modify more stuff on the page but it would be nice if we could have more isolation without sacrificing functionality. I'm still looking into why my more feature-full scripts are not having any effect on the page while running in a separate world. Userscripts are added to both the default and private profile because I that if people have scripts installed they want them to run in private mode too. We are grabbing the scripts from the greasemonkey module, as opposed to reading them directly from disk, because the module adds some GM_* functions that the scripts may expect, and because that is used for webkit anyway. I have code to support qtwebengine <5.8 but didn't because I am not happy with the timing of some of the signals that we are provided regarding page load state, and the actual load state. While the difference between document-end and document-idle isn't so bad, injecting document-start scripts when a urlChanged event is emitted results in the script being injected into the environment for the page being navigated away from. Anyway, if anyone wants this for earlier webengines I can oblige them. --- qutebrowser/browser/webengine/webenginetab.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 89ba958a7..c8d7ef670 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -69,6 +69,10 @@ def init(): download_manager.install(webenginesettings.private_profile) objreg.register('webengine-download-manager', download_manager) + greasemonkey = objreg.get('greasemonkey') + greasemonkey.scripts_reloaded.connect(inject_userscripts) + inject_userscripts() + # Mapping worlds from usertypes.JsWorld to QWebEngineScript world IDs. _JS_WORLD_MAP = { @@ -79,6 +83,42 @@ _JS_WORLD_MAP = { } +def inject_userscripts(): + """Register user javascript files with the global profiles.""" + # The greasemonkey metadata block support in qtwebengine only starts at 5.8 + # Otherwise have to handle injecting the scripts into the page at very + # early load, probs same place in view as the enableJS check. + if not qtutils.version_check('5.8'): + return + + # Since we are inserting scripts into profile.scripts they won't + # just get replaced by new gm scripts like if we were injecting them + # ourselves so we need to remove all gm scripts, while not removing + # any other stuff that might have been added. Like the one for + # stylsheets. + # Could either use a different world for gm scripts, check for gm metadata + # values (would mean no non-gm userscripts), or check the code for + # _qute_script_id + for profile in [webenginesettings.default_profile, + webenginesettings.private_profile]: + scripts = profile.scripts() + for script in scripts.toList(): + if script.worldId() == QWebEngineScript.MainWorld: + scripts.remove(script) + + # Should we be adding to private profile too? + for profile in [webenginesettings.default_profile, + webenginesettings.private_profile]: + scripts = profile.scripts() + greasemonkey = objreg.get('greasemonkey') + for script in greasemonkey.all_scripts(): + new_script = QWebEngineScript() + new_script.setWorldId(QWebEngineScript.MainWorld) + new_script.setSourceCode(script.code()) + log.greasemonkey.debug('adding script: %s', new_script.name) + scripts.insert(new_script) + + class WebEngineAction(browsertab.AbstractAction): """QtWebEngine implementations related to web actions.""" From 325c595b896ded73d324b8a2a9c178b5ef1fd604 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 27 Jul 2017 19:25:29 +1200 Subject: [PATCH 027/322] Greasemonkey: Don't strip gm metadata from scripts when loading. Since we just pass them to webenginescriptcollection on that backend and that wants to parse it itself to figure out injection point etc. --- qutebrowser/browser/greasemonkey.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 058a2d0f4..a616891d1 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -31,8 +31,6 @@ from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.utils import log, standarddir from qutebrowser.commands import cmdutils -# TODO: GM_ bootstrap - def _scripts_dir(): """Get the directory of the scripts.""" @@ -186,7 +184,7 @@ unsafeWindow = window; props, _code = matches except ValueError: props = "" - script = cls(re.findall(cls.PROPS_REGEX, props), code) + script = cls(re.findall(cls.PROPS_REGEX, props), source) script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:])) return script From 799730f6864ad8f41365542d5c393ecbd52f7155 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Thu, 27 Jul 2017 21:21:21 +1200 Subject: [PATCH 028/322] Remove GM_ and userscript variables from global scope. --- qutebrowser/browser/greasemonkey.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index a616891d1..41d32bcf1 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -201,7 +201,8 @@ unsafeWindow = window; scriptName=self.name, scriptInfo=self._meta_json(), scriptMeta=self.script_meta) - return '\n'.join([gm_bootstrap, self._code]) + return '\n'.join( + ["(function(){", gm_bootstrap, self._code, "})();"]) def _meta_json(self): return json.dumps({ From 41035cb5cab72ab3c4acfb06330eab6e7b8f25b4 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sun, 5 Nov 2017 16:36:09 +1300 Subject: [PATCH 029/322] Greasemonkey: restrict page schemes that scripts can run on Scripts shouldn't run on qute://settings or source:// etc. Whitelist from: https://wiki.greasespot.net/Include_and_exclude_rules --- qutebrowser/browser/greasemonkey.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 41d32bcf1..4a8d9bdeb 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -226,6 +226,10 @@ class GreasemonkeyManager(QObject): """ scripts_reloaded = pyqtSignal() + # https://wiki.greasespot.net/Include_and_exclude_rules#Greaseable_schemes + # Limit the schemes scripts can run on due to unreasonable levels of + # exploitability + greaseable_schemes = ['http', 'https', 'ftp', 'file'] def __init__(self, parent=None): super().__init__(parent) @@ -273,6 +277,8 @@ class GreasemonkeyManager(QObject): returns a tuple of lists of scripts meant to run at (document-start, document-end, document-idle) """ + if url.split(':', 1)[0] not in self.greaseable_schemes: + return [], [], [] match = functools.partial(fnmatch.fnmatch, url) tester = (lambda script: any([match(pat) for pat in script.includes]) and From edf737ff7d604a3d14a26eb23b0b0a94f9328230 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 4 Oct 2017 20:42:10 +1300 Subject: [PATCH 030/322] Greasemonkey: move scripts for a domain into data class. Also makes scripts that don't include a greasemonkey metadata block match any url. QWebEngine already has that behaviour. --- qutebrowser/browser/greasemonkey.py | 18 ++++++++++++++++-- qutebrowser/browser/webkit/webpage.py | 10 +++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 4a8d9bdeb..d06bc9cac 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -26,6 +26,7 @@ import fnmatch import functools import glob +import attr from PyQt5.QtCore import pyqtSignal, QObject from qutebrowser.utils import log, standarddir @@ -186,6 +187,8 @@ unsafeWindow = window; props = "" script = cls(re.findall(cls.PROPS_REGEX, props), source) script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:])) + if not props: + script.includes = ['*'] return script def code(self): @@ -215,6 +218,16 @@ unsafeWindow = window; }) +@attr.s +class MatchingScripts(object): + """All userscripts registered to run on a particular url.""" + + url = attr.ib() + start = attr.ib(default=attr.Factory(list)) + end = attr.ib(default=attr.Factory(list)) + idle = attr.ib(default=attr.Factory(list)) + + class GreasemonkeyManager(QObject): """Manager of userscripts and a greasemonkey compatible environment. @@ -278,12 +291,13 @@ class GreasemonkeyManager(QObject): document-end, document-idle) """ if url.split(':', 1)[0] not in self.greaseable_schemes: - return [], [], [] + return MatchingScripts(url, [], [], []) match = functools.partial(fnmatch.fnmatch, url) tester = (lambda script: any([match(pat) for pat in script.includes]) and not any([match(pat) for pat in script.excludes])) - return ( + return MatchingScripts( + url, [script for script in self._run_start if tester(script)], [script for script in self._run_end if tester(script)], [script for script in self._run_idle if tester(script)] diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 095e41fe2..15f662e61 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -300,14 +300,14 @@ class BrowserPage(QWebPage): """ greasemonkey = objreg.get('greasemonkey') url = self.currentFrame().url() - start_scripts, end_scripts, idle_scripts = \ - greasemonkey.scripts_for(url.toDisplayString()) + scripts = greasemonkey.scripts_for(url.toDisplayString()) + if load == "start": - toload = start_scripts + toload = scripts.start elif load == "end": - toload = end_scripts + toload = scripts.end elif load == "idle": - toload = idle_scripts + toload = scripts.idle for script in toload: log.webview.debug('Running GM script: {}'.format(script.name)) From c1b912f5670756b6a96ba378e60e83e9df64ba98 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 4 Oct 2017 21:24:16 +1300 Subject: [PATCH 031/322] Greasemonkey: move inject_userscripts into webenginesettings --- .../browser/webengine/webenginesettings.py | 33 +++++++++++++++ qutebrowser/browser/webengine/webenginetab.py | 40 +------------------ 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 4bf525c46..5ce4e1a05 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -244,6 +244,39 @@ def _init_profiles(): private_profile.setSpellCheckEnabled(True) +def inject_userscripts(): + """Register user javascript files with the global profiles.""" + # The greasemonkey metadata block support in qtwebengine only starts at 5.8 + # Otherwise have to handle injecting the scripts into the page at very + # early load, probs same place in view as the enableJS check. + if not qtutils.version_check('5.8'): + return + + # Since we are inserting scripts into profile.scripts they won't + # just get replaced by new gm scripts like if we were injecting them + # ourselves so we need to remove all gm scripts, while not removing + # any other stuff that might have been added. Like the one for + # stylsheets. + # Could either use a different world for gm scripts, check for gm metadata + # values (would mean no non-gm userscripts), or check the code for + # _qute_script_id + for profile in [default_profile, private_profile]: + scripts = profile.scripts() + for script in scripts.toList(): + if script.worldId() == QWebEngineScript.MainWorld: + scripts.remove(script) + + for profile in [default_profile, private_profile]: + scripts = profile.scripts() + greasemonkey = objreg.get('greasemonkey') + for script in greasemonkey.all_scripts(): + new_script = QWebEngineScript() + new_script.setWorldId(QWebEngineScript.MainWorld) + new_script.setSourceCode(script.code()) + log.greasemonkey.debug('adding script: %s', new_script.name()) + scripts.insert(new_script) + + def init(args): """Initialize the global QWebSettings.""" if args.enable_webengine_inspector: diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index c8d7ef670..c97aaacbe 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -70,8 +70,8 @@ def init(): objreg.register('webengine-download-manager', download_manager) greasemonkey = objreg.get('greasemonkey') - greasemonkey.scripts_reloaded.connect(inject_userscripts) - inject_userscripts() + greasemonkey.scripts_reloaded.connect(webenginesettings.inject_userscripts) + webenginesettings.inject_userscripts() # Mapping worlds from usertypes.JsWorld to QWebEngineScript world IDs. @@ -83,42 +83,6 @@ _JS_WORLD_MAP = { } -def inject_userscripts(): - """Register user javascript files with the global profiles.""" - # The greasemonkey metadata block support in qtwebengine only starts at 5.8 - # Otherwise have to handle injecting the scripts into the page at very - # early load, probs same place in view as the enableJS check. - if not qtutils.version_check('5.8'): - return - - # Since we are inserting scripts into profile.scripts they won't - # just get replaced by new gm scripts like if we were injecting them - # ourselves so we need to remove all gm scripts, while not removing - # any other stuff that might have been added. Like the one for - # stylsheets. - # Could either use a different world for gm scripts, check for gm metadata - # values (would mean no non-gm userscripts), or check the code for - # _qute_script_id - for profile in [webenginesettings.default_profile, - webenginesettings.private_profile]: - scripts = profile.scripts() - for script in scripts.toList(): - if script.worldId() == QWebEngineScript.MainWorld: - scripts.remove(script) - - # Should we be adding to private profile too? - for profile in [webenginesettings.default_profile, - webenginesettings.private_profile]: - scripts = profile.scripts() - greasemonkey = objreg.get('greasemonkey') - for script in greasemonkey.all_scripts(): - new_script = QWebEngineScript() - new_script.setWorldId(QWebEngineScript.MainWorld) - new_script.setSourceCode(script.code()) - log.greasemonkey.debug('adding script: %s', new_script.name) - scripts.insert(new_script) - - class WebEngineAction(browsertab.AbstractAction): """QtWebEngine implementations related to web actions.""" From fd5d44182ba53e6e3c5fb8c71ffe15d3c44aeeaa Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 4 Oct 2017 22:39:32 +1300 Subject: [PATCH 032/322] Greasemonkey: move GM_* template into seperate file. Also ported it to jinja rather than str.format(). Also ran the js through jslint and fixed up a few very minor things. --- qutebrowser/browser/greasemonkey.py | 123 +----------------- qutebrowser/javascript/.eslintignore | 2 + .../javascript/greasemonkey_wrapper.js | 118 +++++++++++++++++ qutebrowser/utils/jinja.py | 1 + 4 files changed, 128 insertions(+), 116 deletions(-) create mode 100644 qutebrowser/javascript/greasemonkey_wrapper.js diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index d06bc9cac..e1f0b57db 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -29,7 +29,7 @@ import glob import attr from PyQt5.QtCore import pyqtSignal, QObject -from qutebrowser.utils import log, standarddir +from qutebrowser.utils import log, standarddir, jinja from qutebrowser.commands import cmdutils @@ -41,115 +41,6 @@ def _scripts_dir(): class GreasemonkeyScript: """Container class for userscripts, parses metadata blocks.""" - GM_BOOTSTRAP_TEMPLATE = r"""var _qute_script_id = "__gm_{scriptName}"; - -function GM_log(text) {{ - console.log(text); -}} - -GM_info = (function() {{ - return {{ - 'script': {scriptInfo}, - 'scriptMetaStr': {scriptMeta}, - 'scriptWillUpdate': false, - 'version': '0.0.1', - 'scriptHandler': 'Tampermonkey' //so scripts don't expect exportFunction - }}; -}}()); - -function GM_setValue(key, value) {{ - if (localStorage !== null && - typeof key === "string" && - (typeof value === "string" || - typeof value === "number" || - typeof value == "boolean")) {{ - localStorage.setItem(_qute_script_id + key, value); - }} -}} - -function GM_getValue(key, default_) {{ - if (localStorage !== null && typeof key === "string") {{ - return localStorage.getItem(_qute_script_id + key) || default_; - }} -}} - -function GM_deleteValue(key) {{ - if (localStorage !== null && typeof key === "string") {{ - localStorage.removeItem(_qute_script_id + key); - }} -}} - -function GM_listValues() {{ - var i; - var keys = []; - for (i = 0; i < localStorage.length; ++i) {{ - if (localStorage.key(i).startsWith(_qute_script_id)) {{ - keys.push(localStorage.key(i)); - }} - }} - return keys; -}} - -function GM_openInTab(url) {{ - window.open(url); -}} - - -// Almost verbatim copy from Eric -function GM_xmlhttpRequest(/* object */ details) {{ - details.method = details.method.toUpperCase() || "GET"; - - if(!details.url) {{ - throw("GM_xmlhttpRequest requires an URL."); - }} - - // build XMLHttpRequest object - var oXhr = new XMLHttpRequest; - // run it - if("onreadystatechange" in details) - oXhr.onreadystatechange = function() {{ - details.onreadystatechange(oXhr) - }}; - if("onload" in details) - oXhr.onload = function() {{ details.onload(oXhr) }}; - if("onerror" in details) - oXhr.onerror = function() {{ details.onerror(oXhr) }}; - - oXhr.open(details.method, details.url, true); - - if("headers" in details) - for(var header in details.headers) - oXhr.setRequestHeader(header, details.headers[header]); - - if("data" in details) - oXhr.send(details.data); - else - oXhr.send(); -}} - -function GM_addStyle(/* String */ styles) {{ - var head = document.getElementsByTagName("head")[0]; - if (head === undefined) {{ - document.onreadystatechange = function() {{ - if (document.readyState == "interactive") {{ - var oStyle = document.createElement("style"); - oStyle.setAttribute("type", "text/css"); - oStyle.appendChild(document.createTextNode(styles)); - document.getElementsByTagName("head")[0].appendChild(oStyle); - }} - }} - }} - else {{ - var oStyle = document.createElement("style"); - oStyle.setAttribute("type", "text/css"); - oStyle.appendChild(document.createTextNode(styles)); - head.appendChild(oStyle); - }} -}} - -unsafeWindow = window; -""" - def __init__(self, properties, code): self._code = code self.includes = [] @@ -200,12 +91,12 @@ unsafeWindow = window; browser's debugger/inspector will not match up to the line numbers in the source script directly. """ - gm_bootstrap = self.GM_BOOTSTRAP_TEMPLATE.format( - scriptName=self.name, - scriptInfo=self._meta_json(), - scriptMeta=self.script_meta) - return '\n'.join( - ["(function(){", gm_bootstrap, self._code, "})();"]) + return jinja.js_environment.get_template( + 'greasemonkey_wrapper.js').render( + scriptName=self.name, + scriptInfo=self._meta_json(), + scriptMeta=self.script_meta, + scriptSource=self._code) def _meta_json(self): return json.dumps({ diff --git a/qutebrowser/javascript/.eslintignore b/qutebrowser/javascript/.eslintignore index ca4d3c667..036a72cfe 100644 --- a/qutebrowser/javascript/.eslintignore +++ b/qutebrowser/javascript/.eslintignore @@ -1,2 +1,4 @@ # Upstream Mozilla's code pac_utils.js +# Actually a jinja template so eslint chokes on the {{}} syntax. +greasemonkey_wrapper.js diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js new file mode 100644 index 000000000..b49dc2c02 --- /dev/null +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -0,0 +1,118 @@ +(function () { + var _qute_script_id = "__gm_{{ scriptName }}"; + + function GM_log(text) { + console.log(text); + } + + var GM_info = (function () { + return { + 'script': {{ scriptInfo }}, + 'scriptMetaStr': {{ scriptMeta }}, + 'scriptWillUpdate': false, + 'version': '0.0.1', + 'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction + }; + }()); + + function GM_setValue(key, value) { + if (localStorage !== null && + typeof key === "string" && + (typeof value === "string" || + typeof value === "number" || + typeof value === "boolean")) { + localStorage.setItem(_qute_script_id + key, value); + } + } + + function GM_getValue(key, default_) { + if (localStorage !== null && typeof key === "string") { + return localStorage.getItem(_qute_script_id + key) || default_; + } + } + + function GM_deleteValue(key) { + if (localStorage !== null && typeof key === "string") { + localStorage.removeItem(_qute_script_id + key); + } + } + + function GM_listValues() { + var i, keys = []; + for (i = 0; i < localStorage.length; i = i + 1) { + if (localStorage.key(i).startsWith(_qute_script_id)) { + keys.push(localStorage.key(i)); + } + } + return keys; + } + + function GM_openInTab(url) { + window.open(url); + } + + + // Almost verbatim copy from Eric + function GM_xmlhttpRequest(/* object */ details) { + details.method = details.method.toUpperCase() || "GET"; + + if (!details.url) { + throw ("GM_xmlhttpRequest requires an URL."); + } + + // build XMLHttpRequest object + var oXhr = new XMLHttpRequest(); + // run it + if ("onreadystatechange" in details) { + oXhr.onreadystatechange = function () { + details.onreadystatechange(oXhr); + }; + } + if ("onload" in details) { + oXhr.onload = function () { details.onload(oXhr) }; + } + if ("onerror" in details) { + oXhr.onerror = function () { details.onerror(oXhr) }; + } + + oXhr.open(details.method, details.url, true); + + if ("headers" in details) { + for (var header in details.headers) { + oXhr.setRequestHeader(header, details.headers[header]); + } + } + + if ("data" in details) { + oXhr.send(details.data); + } else { + oXhr.send(); + } + } + + function GM_addStyle(/* String */ styles) { + var head = document.getElementsByTagName("head")[0]; + if (head === undefined) { + document.onreadystatechange = function () { + if (document.readyState == "interactive") { + var oStyle = document.createElement("style"); + oStyle.setAttribute("type", "text/css"); + oStyle.appendChild(document.createTextNode(styles)); + document.getElementsByTagName("head")[0].appendChild(oStyle); + } + } + } + else { + var oStyle = document.createElement("style"); + oStyle.setAttribute("type", "text/css"); + oStyle.appendChild(document.createTextNode(styles)); + head.appendChild(oStyle); + } + } + + unsafeWindow = window; + + //====== The actual user script source ======// +{{ scriptSource }} + //====== End User Script ======// +})(); diff --git a/qutebrowser/utils/jinja.py b/qutebrowser/utils/jinja.py index e7b536b60..b6f53645b 100644 --- a/qutebrowser/utils/jinja.py +++ b/qutebrowser/utils/jinja.py @@ -136,3 +136,4 @@ def render(template, **kwargs): environment = Environment() +js_environment = jinja2.Environment(loader=Loader('javascript')) From a7f41b4564dd39247893662ebeb23170a290c85f Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 4 Oct 2017 23:25:22 +1300 Subject: [PATCH 033/322] Greasemonkey: ensure only GM scripts are cleaned up on reload. WebEngine only. Previously we were just removing every script from the main world. But some other scripts might got here in the future so new we are overriding the name field to add a GM- prefix so hopefully we only remove greasemonkey scripts before adding new ones. --- qutebrowser/browser/webengine/webenginesettings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 5ce4e1a05..18ba98fd6 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -263,7 +263,9 @@ def inject_userscripts(): for profile in [default_profile, private_profile]: scripts = profile.scripts() for script in scripts.toList(): - if script.worldId() == QWebEngineScript.MainWorld: + if script.name().startswith("GM-"): + log.greasemonkey.debug('removing script: {}' + .format(script.name())) scripts.remove(script) for profile in [default_profile, private_profile]: @@ -273,6 +275,7 @@ def inject_userscripts(): new_script = QWebEngineScript() new_script.setWorldId(QWebEngineScript.MainWorld) new_script.setSourceCode(script.code()) + new_script.setName("GM-{}".format(script.name)) log.greasemonkey.debug('adding script: %s', new_script.name()) scripts.insert(new_script) From d93c583c0d8f1859be2e306f21477ee2444b2d83 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 7 Oct 2017 17:18:48 +1300 Subject: [PATCH 034/322] Greasemonkey: Escape jinja variables for JS strings. --- qutebrowser/javascript/greasemonkey_wrapper.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index b49dc2c02..a5079b89a 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -1,5 +1,5 @@ (function () { - var _qute_script_id = "__gm_{{ scriptName }}"; + var _qute_script_id = "__gm_"+{{ scriptName | tojson }}; function GM_log(text) { console.log(text); @@ -7,8 +7,8 @@ var GM_info = (function () { return { - 'script': {{ scriptInfo }}, - 'scriptMetaStr': {{ scriptMeta }}, + 'script': {{ scriptInfo | tojson }}, + 'scriptMetaStr': {{ scriptMeta | tojson }}, 'scriptWillUpdate': false, 'version': '0.0.1', 'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction From 5e49e7eef21ef1abaf327b1c060145f67ba6f868 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 7 Oct 2017 17:22:57 +1300 Subject: [PATCH 035/322] Greasemonkey: Throw Errors if GM_ function args wrong type. These argument type restrictions are mentioned on the greasespot pages for these value storage functions. We could call JSON.dumps() instead but better to push that onto the caller so we don't have to try handle deserialization. Also removes the check for localstorage because everyone has supported that for years. --- .../javascript/greasemonkey_wrapper.js | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index a5079b89a..605f82d5a 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -16,25 +16,29 @@ }()); function GM_setValue(key, value) { - if (localStorage !== null && - typeof key === "string" && - (typeof value === "string" || - typeof value === "number" || - typeof value === "boolean")) { - localStorage.setItem(_qute_script_id + key, value); + if (typeof key !== "string") { + throw new Error("GM_setValue requires the first parameter to be of type string, not '"+typeof key+"'"); } + if (typeof value !== "string" || + typeof value !== "number" || + typeof value !== "boolean") { + throw new Error("GM_setValue requires the second parameter to be of type string, number or boolean, not '"+typeof value+"'"); + } + localStorage.setItem(_qute_script_id + key, value); } function GM_getValue(key, default_) { - if (localStorage !== null && typeof key === "string") { - return localStorage.getItem(_qute_script_id + key) || default_; + if (typeof key !== "string") { + throw new Error("GM_getValue requires the first parameter to be of type string, not '"+typeof key+"'"); } + return localStorage.getItem(_qute_script_id + key) || default_; } function GM_deleteValue(key) { - if (localStorage !== null && typeof key === "string") { - localStorage.removeItem(_qute_script_id + key); + if (typeof key !== "string") { + throw new Error("GM_deleteValue requires the first parameter to be of type string, not '"+typeof key+"'"); } + localStorage.removeItem(_qute_script_id + key); } function GM_listValues() { From d318178567f8e4236ded6ff350628127d4be7496 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 7 Oct 2017 17:32:21 +1300 Subject: [PATCH 036/322] Greasemonkey: Fix metadata block regex. This regex was broken since the original PR and subsequent code seemed to be working around it. Before re.split was returning [everything up to /UserScript, everything else], now it returns [before UserScript, metadata, after /UserScript], which is good. Also I added the check for the UserScript line starting at column 0 as per spec. --- qutebrowser/browser/greasemonkey.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index e1f0b57db..a0eb109fd 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -60,7 +60,7 @@ class GreasemonkeyScript: elif name == 'run-at': self.run_at = value - HEADER_REGEX = r'// ==UserScript==.|\n+// ==/UserScript==\n' + HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n' PROPS_REGEX = r'// @(?P[^\s]+)\s+(?P.+)' @classmethod @@ -71,13 +71,13 @@ class GreasemonkeyScript: Parses the greasemonkey metadata block, if present, to fill out attributes. """ - matches = re.split(cls.HEADER_REGEX, source, maxsplit=1) + matches = re.split(cls.HEADER_REGEX, source, maxsplit=2) try: - props, _code = matches + _, props, _code = matches except ValueError: props = "" script = cls(re.findall(cls.PROPS_REGEX, props), source) - script.script_meta = '"{}"'.format("\\n".join(props.split('\n')[2:])) + script.script_meta = props if not props: script.includes = ['*'] return script From 209e43e0baa0eda4c3258abc2dad75618020db99 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 7 Oct 2017 17:33:00 +1300 Subject: [PATCH 037/322] Greasemonkey: Match against percent encoded urls only. This change requires urls specified in @include, @exclude and @matches directives in metadata blocks to be in the same form that QUrl.toEncoded() returns. That is a punycoded domain and percent encoded path and query. This seems to be what Tampermonkey on chrome expects to. Also changes the scripts_for() function to take a QUrl arg so the caller doesn't need to worry about encodings. --- qutebrowser/browser/greasemonkey.py | 5 +++-- qutebrowser/browser/webkit/webpage.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index a0eb109fd..3243ed8d8 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -181,9 +181,10 @@ class GreasemonkeyManager(QObject): returns a tuple of lists of scripts meant to run at (document-start, document-end, document-idle) """ - if url.split(':', 1)[0] not in self.greaseable_schemes: + if url.scheme() not in self.greaseable_schemes: return MatchingScripts(url, [], [], []) - match = functools.partial(fnmatch.fnmatch, url) + match = functools.partial(fnmatch.fnmatch, + str(url.toEncoded(), 'utf-8')) tester = (lambda script: any([match(pat) for pat in script.includes]) and not any([match(pat) for pat in script.excludes])) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 15f662e61..e48519189 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -300,7 +300,7 @@ class BrowserPage(QWebPage): """ greasemonkey = objreg.get('greasemonkey') url = self.currentFrame().url() - scripts = greasemonkey.scripts_for(url.toDisplayString()) + scripts = greasemonkey.scripts_for(url) if load == "start": toload = scripts.start From efde31aa5700f41b75ee7f119d6c3cacea9cdf90 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sun, 8 Oct 2017 12:35:01 +1300 Subject: [PATCH 038/322] Greasemonkey: Support QTWebEngine versions < 5.8 QTWebEngine 5.8 added support for parsing greasemonkey metadata blocks and scripts added to the QWebEngineScriptCollection of a page or its profile and then deciding what urls to run those scripts on and at what point in the load process to run them. For earlier versions we must do that work ourselves. But with the additional handicap of the less rich qtwebengine api. We have acceptNavigationRequest, loadStarted, loadProgress, loadFinished, urlChanged to choose from regarding points at which to register scripts for the current page. Adding scripts on acceptNavigation loadStarted and loadFinished causes scripts to run too early or too late (eg on the pages being navigated from/to) and not run on the desired page at the time they are inserted. We could maybe do some more sophisticated stuff with loadProgress but it didn't have any better behaviour in the brief testing I gave it. Registering scripts on the urlChanged event seems to work fine. Even if it seems like there could be problems with the signal firing too often, due to not necessarily being tied to the page load progress, that doesn't seem to have an effect in practice. The event is fired when, for example, the url fragment changes and even if we add a new script to the collection (or remove an existing one) it doesn't have an effect on what is running on the page. I suspect all of those timing issues is due to the signals being forwarded fairly directly from the underlying chomium/blink code but the webengine script stuff only being pushed back to the implementation on certain events. Anyway, using urlChanged seems to work fine due to some quirk(s) of the implementation. That might change with later development but this codepath is only ever going to be used for version 5.7. There are other potential optimizations like not removing and then re-adding scripts for the current page. But they probably wouldn't do anything anyway, or at least anything that you would expect. --- qutebrowser/browser/webengine/webview.py | 45 ++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index 56bd1eb5a..35d6fca40 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -23,12 +23,14 @@ import functools from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, PYQT_VERSION from PyQt5.QtGui import QPalette -from PyQt5.QtWebEngineWidgets import QWebEngineView, QWebEnginePage +from PyQt5.QtWebEngineWidgets import (QWebEngineView, QWebEnginePage, + QWebEngineScript) from qutebrowser.browser import shared from qutebrowser.browser.webengine import certificateerror, webenginesettings from qutebrowser.config import config -from qutebrowser.utils import log, debug, usertypes, jinja, urlutils, message +from qutebrowser.utils import (log, debug, usertypes, jinja, urlutils, message, + objreg, qtutils) class WebEngineView(QWebEngineView): @@ -135,6 +137,7 @@ class WebEnginePage(QWebEnginePage): self._theme_color = theme_color self._set_bg_color() config.instance.changed.connect(self._set_bg_color) + self.urlChanged.connect(self._inject_userjs) @config.change_filter('colors.webpage.bg') def _set_bg_color(self): @@ -300,3 +303,41 @@ class WebEnginePage(QWebEnginePage): message.error(msg) return False return True + + @pyqtSlot('QUrl') + def _inject_userjs(self, url): + """Inject userscripts registered for `url` into the current page.""" + if qtutils.version_check('5.8'): + # Handled in webenginetab with the builtin greasemonkey + # support. + return + + # Using QWebEnginePage.scripts() to hold the user scripts means + # we don't have to worry ourselves about where to inject the + # page but also means scripts hang around for the tab lifecycle. + # So clear them here. + scripts = self.scripts() + for script in scripts.toList(): + if script.name().startswith("GM-"): + really_removed = scripts.remove(script) + log.greasemonkey.debug("Removing ({}) script: {}" + .format(really_removed, script.name())) + + def _add_script(script, injection_point): + new_script = QWebEngineScript() + new_script.setInjectionPoint(injection_point) + new_script.setWorldId(QWebEngineScript.MainWorld) + new_script.setSourceCode(script.code()) + new_script.setName("GM-{}".format(script.name)) + log.greasemonkey.debug("Adding script: {}" + .format(new_script.name())) + scripts.insert(new_script) + + greasemonkey = objreg.get('greasemonkey') + matching_scripts = greasemonkey.scripts_for(url) + for script in matching_scripts.start: + _add_script(script, QWebEngineScript.DocumentCreation) + for script in matching_scripts.end: + _add_script(script, QWebEngineScript.DocumentReady) + for script in matching_scripts.idle: + _add_script(script, QWebEngineScript.Deferred) From fb019b2dab342a98ae8e364e893650fecd42fd62 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 10 Oct 2017 20:45:10 +1300 Subject: [PATCH 039/322] Address second round line comments. Add qute version to GM_info object in GM wrapper. Support using the greasemonkey @namespace metadata for its intended purpose of avoiding name collisions. Get a nice utf8 encoded string from a QUrl more better. --- qutebrowser/browser/greasemonkey.py | 9 +++++--- .../javascript/greasemonkey_wrapper.js | 22 +++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 3243ed8d8..b58a558a5 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -27,7 +27,7 @@ import functools import glob import attr -from PyQt5.QtCore import pyqtSignal, QObject +from PyQt5.QtCore import pyqtSignal, QObject, QUrl from qutebrowser.utils import log, standarddir, jinja from qutebrowser.commands import cmdutils @@ -47,10 +47,13 @@ class GreasemonkeyScript: self.excludes = [] self.description = None self.name = None + self.namespace = None self.run_at = None for name, value in properties: if name == 'name': self.name = value + elif name == 'namespace': + self.namespace = value elif name == 'description': self.description = value elif name in ['include', 'match']: @@ -93,7 +96,7 @@ class GreasemonkeyScript: """ return jinja.js_environment.get_template( 'greasemonkey_wrapper.js').render( - scriptName=self.name, + scriptName="/".join([self.namespace or '', self.name]), scriptInfo=self._meta_json(), scriptMeta=self.script_meta, scriptSource=self._code) @@ -184,7 +187,7 @@ class GreasemonkeyManager(QObject): if url.scheme() not in self.greaseable_schemes: return MatchingScripts(url, [], [], []) match = functools.partial(fnmatch.fnmatch, - str(url.toEncoded(), 'utf-8')) + url.toString(QUrl.FullyEncoded)) tester = (lambda script: any([match(pat) for pat in script.includes]) and not any([match(pat) for pat in script.excludes])) diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index 605f82d5a..eb2d8fea1 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -5,15 +5,19 @@ console.log(text); } - var GM_info = (function () { - return { - 'script': {{ scriptInfo | tojson }}, - 'scriptMetaStr': {{ scriptMeta | tojson }}, - 'scriptWillUpdate': false, - 'version': '0.0.1', - 'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction - }; - }()); + var GM_info = { + 'script': {{ scriptInfo }}, + 'scriptMetaStr': {{ scriptMeta | tojson }}, + 'scriptWillUpdate': false, + 'version': "0.0.1", + 'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction + }; + + function checkKey(key, funcName) { + if (typeof key !== "string") { + throw new Error(funcName+" requires the first parameter to be of type string, not '"+typeof key+"'"); + } + } function GM_setValue(key, value) { if (typeof key !== "string") { From c0832eb04b87b6762f4d84440039bc6cf8dc472b Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 10 Oct 2017 20:52:41 +1300 Subject: [PATCH 040/322] Greasemonkey: support @nosubframes. And run on frames by default. At least on webengine. There is probably some api to enumerate frames on a webkit page. Not tested. --- qutebrowser/browser/greasemonkey.py | 5 +++++ qutebrowser/browser/webengine/webenginesettings.py | 4 +++- qutebrowser/browser/webengine/webview.py | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index b58a558a5..cd6fce27a 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -49,6 +49,9 @@ class GreasemonkeyScript: self.name = None self.namespace = None self.run_at = None + self.script_meta = None + # Running on subframes is only supported on the qtwebengine backend. + self.runs_on_sub_frames = True for name, value in properties: if name == 'name': self.name = value @@ -62,6 +65,8 @@ class GreasemonkeyScript: self.excludes.append(value) elif name == 'run-at': self.run_at = value + elif name == 'noframes': + self.runs_on_sub_frames = False HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n' PROPS_REGEX = r'// @(?P[^\s]+)\s+(?P.+)' diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 18ba98fd6..dade671c8 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -276,7 +276,9 @@ def inject_userscripts(): new_script.setWorldId(QWebEngineScript.MainWorld) new_script.setSourceCode(script.code()) new_script.setName("GM-{}".format(script.name)) - log.greasemonkey.debug('adding script: %s', new_script.name()) + new_script.setRunsOnSubFrames(script.runs_on_sub_frames) + log.greasemonkey.debug('adding script: {}' + .format(new_script.name())) scripts.insert(new_script) diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index 35d6fca40..af4476d4a 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -329,6 +329,7 @@ class WebEnginePage(QWebEnginePage): new_script.setWorldId(QWebEngineScript.MainWorld) new_script.setSourceCode(script.code()) new_script.setName("GM-{}".format(script.name)) + new_script.setRunsOnSubFrames(script.runs_on_sub_frames) log.greasemonkey.debug("Adding script: {}" .format(new_script.name())) scripts.insert(new_script) From 7c497427ce7cdaf3d23362cdae570b9660d7cf45 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 10 Oct 2017 20:54:58 +1300 Subject: [PATCH 041/322] Greasemonkey: various javascript fixups to GM wrapper template. Thanks to @sandrosc. A few breaking changes fixed (default method to GM_xhr not working, GM_listvalues not cleaning up output, GM_setvalue param checking logic wrong) and a few hygenic changes made. --- .../javascript/greasemonkey_wrapper.js | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index eb2d8fea1..7a271375d 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -20,11 +20,9 @@ } function GM_setValue(key, value) { - if (typeof key !== "string") { - throw new Error("GM_setValue requires the first parameter to be of type string, not '"+typeof key+"'"); - } - if (typeof value !== "string" || - typeof value !== "number" || + checkKey(key, "GM_setValue"); + if (typeof value !== "string" && + typeof value !== "number" && typeof value !== "boolean") { throw new Error("GM_setValue requires the second parameter to be of type string, number or boolean, not '"+typeof value+"'"); } @@ -32,24 +30,20 @@ } function GM_getValue(key, default_) { - if (typeof key !== "string") { - throw new Error("GM_getValue requires the first parameter to be of type string, not '"+typeof key+"'"); - } + checkKey(key, "GM_getValue"); return localStorage.getItem(_qute_script_id + key) || default_; } function GM_deleteValue(key) { - if (typeof key !== "string") { - throw new Error("GM_deleteValue requires the first parameter to be of type string, not '"+typeof key+"'"); - } + checkKey(key, "GM_deleteValue"); localStorage.removeItem(_qute_script_id + key); } function GM_listValues() { - var i, keys = []; - for (i = 0; i < localStorage.length; i = i + 1) { + var keys = []; + for (var i = 0; i < localStorage.length; i++) { if (localStorage.key(i).startsWith(_qute_script_id)) { - keys.push(localStorage.key(i)); + keys.push(localStorage.key(i).slice(_qute_script_id.length)); } } return keys; @@ -62,7 +56,7 @@ // Almost verbatim copy from Eric function GM_xmlhttpRequest(/* object */ details) { - details.method = details.method.toUpperCase() || "GET"; + details.method = details.method ? details.method.toUpperCase() : "GET"; if (!details.url) { throw ("GM_xmlhttpRequest requires an URL."); @@ -99,26 +93,24 @@ } function GM_addStyle(/* String */ styles) { + var oStyle = document.createElement("style"); + oStyle.setAttribute("type", "text/css"); + oStyle.appendChild(document.createTextNode(styles)); + var head = document.getElementsByTagName("head")[0]; if (head === undefined) { document.onreadystatechange = function () { if (document.readyState == "interactive") { - var oStyle = document.createElement("style"); - oStyle.setAttribute("type", "text/css"); - oStyle.appendChild(document.createTextNode(styles)); document.getElementsByTagName("head")[0].appendChild(oStyle); } } } else { - var oStyle = document.createElement("style"); - oStyle.setAttribute("type", "text/css"); - oStyle.appendChild(document.createTextNode(styles)); head.appendChild(oStyle); } } - unsafeWindow = window; + var unsafeWindow = window; //====== The actual user script source ======// {{ scriptSource }} From 4c3461038dc4fe2a0732860f064f13a3d1c9f298 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 10 Oct 2017 20:59:27 +1300 Subject: [PATCH 042/322] Greasemonkey: add minimal end-to-end test. Just runs a greasemonkey script on a test page and uses console.log to ensure it is running. Tests @include, and basic happy path greasemonkey.py operation (loading and parsing script, scrip_for on webkit), only testing document-start injecting point but that is the troublsome one at this point. Tested on py35 debian unstable (oldwebkit and qtwebengine5.9) debian stable qtwebengine5.7. Note the extra :reload call for qt5.7 because document-start scripts don't seem to run on the first page load with the current insertion point. I need to look into this more to look at ways of fixing this. --- tests/end2end/features/javascript.feature | 9 ++++++++ tests/end2end/features/test_javascript_bdd.py | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature index a309d6187..66d108125 100644 --- a/tests/end2end/features/javascript.feature +++ b/tests/end2end/features/javascript.feature @@ -123,3 +123,12 @@ Feature: Javascript stuff And I wait for "[*/data/javascript/windowsize.html:*] loaded" in the log And I run :tab-next Then the window sizes should be the same + + Scenario: Have a greasemonkey script run on a page + When I have a greasemonkey file saved + And I run :greasemonkey-reload + And I open data/title.html + # This second reload is required in webengine < 5.8 for scripts + # registered to run at document-start, some sort of timing issue. + And I run :reload + Then the javascript message "Script is running." should be logged diff --git a/tests/end2end/features/test_javascript_bdd.py b/tests/end2end/features/test_javascript_bdd.py index 9f6c021ce..cb81e1ef6 100644 --- a/tests/end2end/features/test_javascript_bdd.py +++ b/tests/end2end/features/test_javascript_bdd.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . +import os.path + import pytest_bdd as bdd bdd.scenarios('javascript.feature') @@ -29,3 +31,23 @@ def check_window_sizes(quteproc): hidden_size = hidden.message.split()[-1] visible_size = visible.message.split()[-1] assert hidden_size == visible_size + +test_gm_script=""" +// ==UserScript== +// @name Qutebrowser test userscript +// @namespace invalid.org +// @include http://localhost:*/data/title.html +// @exclude ??? +// @run-at document-start +// ==/UserScript== +console.log("Script is running on " + window.location.pathname); +""" + +@bdd.when("I have a greasemonkey file saved") +def create_greasemonkey_file(quteproc): + script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey') + os.mkdir(script_path) + file_path = os.path.join(script_path, 'test.user.js') + with open(file_path, 'w', encoding='utf-8') as f: + f.write(test_gm_script) + From 9aeb5775c1856a612d72e4bb2e791d3a0c50775c Mon Sep 17 00:00:00 2001 From: Jimmy Date: Tue, 24 Oct 2017 21:19:22 +1300 Subject: [PATCH 043/322] greasemonkey: run scripts on subframes on webkit Use the `QWebPage.frameCreated` signal to get notifications of subframes and connect the javascript injection triggering signals on those frames too. I had to add a `url = url() or requestedUrl()` bit in there because the inject_userjs method was getting called to early or something when frame.url() wasn't set or was set to the previous page so we were passing the wrong url to greasemonkey.scripts_for(). I ran into a bizarre (I maybe it is completely obvious and I just don't see it) issue where the signals attached to the main frame that were connected to a partial function with the main frame as an argument were not getting emitted, or at least those partial functions were not being called. I worked around it by using None to mean defaulting to the main frame in a couple of places. --- qutebrowser/browser/greasemonkey.py | 1 - qutebrowser/browser/webkit/webpage.py | 63 +++++++++++++++++++++------ 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index cd6fce27a..c8ada7b9a 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -50,7 +50,6 @@ class GreasemonkeyScript: self.namespace = None self.run_at = None self.script_meta = None - # Running on subframes is only supported on the qtwebengine backend. self.runs_on_sub_frames = True for name, value in properties: if name == 'name': diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index e48519189..299815cc3 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -86,12 +86,33 @@ class BrowserPage(QWebPage): self.on_save_frame_state_requested) self.restoreFrameStateRequested.connect( self.on_restore_frame_state_requested) - self.mainFrame().javaScriptWindowObjectCleared.connect( - functools.partial(self.inject_userjs, load='start')) - self.mainFrame().initialLayoutCompleted.connect( - functools.partial(self.inject_userjs, load='end')) - self.mainFrame().loadFinished.connect( - functools.partial(self.inject_userjs, load='idle')) + self.connect_userjs_signals(None) + self.frameCreated.connect(self.connect_userjs_signals) + + @pyqtSlot('QWebFrame*') + def connect_userjs_signals(self, frame_arg): + """ + Connect the signals used as triggers for injecting user + javascripts into `frame_arg`. + """ + # If we pass whatever self.mainFrame() or self.currentFrame() returns + # at init time into the partial functions which the signals + # below call then the signals don't seem to be called at all for + # the main frame of the first tab. I have no idea why I am + # seeing this behavior. Replace the None in the call to this + # function in __init__ with self.mainFrame() and try for + # yourself. + if frame_arg: + frame = frame_arg + else: + frame = self.mainFrame() + + frame.javaScriptWindowObjectCleared.connect( + functools.partial(self.inject_userjs, frame_arg, load='start')) + frame.initialLayoutCompleted.connect( + functools.partial(self.inject_userjs, frame_arg, load='end')) + frame.loadFinished.connect( + functools.partial(self.inject_userjs, frame_arg, load='idle')) def javaScriptPrompt(self, frame, js_msg, default): """Override javaScriptPrompt to use qutebrowser prompts.""" @@ -290,16 +311,24 @@ class BrowserPage(QWebPage): self.error_occurred = False @pyqtSlot() - def inject_userjs(self, load): + def inject_userjs(self, frame, load): """Inject user javascripts into the page. - param: The page load stage to inject the corresponding scripts - for. Support values are "start", "end" and "idle", - corresponding to the allowed values of the `@run-at` - directive in the greasemonkey metadata spec. + Args: + frame: The QWebFrame to inject the user scripts into, or + None for the main frame. + load: The page load stage to inject the corresponding + scripts for. Support values are "start", "end" and + "idle", corresponding to the allowed values of the + `@run-at` directive in the greasemonkey metadata spec. """ + if not frame: + frame = self.mainFrame() + url = frame.url() + if url.isEmpty(): + url = frame.requestedUrl() + greasemonkey = objreg.get('greasemonkey') - url = self.currentFrame().url() scripts = greasemonkey.scripts_for(url) if load == "start": @@ -309,9 +338,15 @@ class BrowserPage(QWebPage): elif load == "idle": toload = scripts.idle + if url.isEmpty(): + # This happens during normal usage like with view source but may + # also indicate a bug. + log.greasemonkey.debug("Not running scripts for frame with no " + "url: {}".format(frame)) for script in toload: - log.webview.debug('Running GM script: {}'.format(script.name)) - self.currentFrame().evaluateJavaScript(script.code()) + if frame is self.mainFrame() or script.runs_on_sub_frames: + log.webview.debug('Running GM script: {}'.format(script.name)) + frame.evaluateJavaScript(script.code()) @pyqtSlot('QWebFrame*', 'QWebPage::Feature') def _on_feature_permission_requested(self, frame, feature): From 361a1ed6e46bf602825986e97c540b17c44b12a7 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 1 Nov 2017 23:37:53 +1300 Subject: [PATCH 044/322] Greasemonkey: change PROPS_REGEX to handle non-value keys. We weren't actually picking up the @noframes greasemonkey directive because of this. I haven't tested this very extensively but it seems to work for making the property value optional. --- qutebrowser/browser/greasemonkey.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index c8ada7b9a..845b83d85 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -68,7 +68,7 @@ class GreasemonkeyScript: self.runs_on_sub_frames = False HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n' - PROPS_REGEX = r'// @(?P[^\s]+)\s+(?P.+)' + PROPS_REGEX = r'// @(?P[^\s]+)\s*(?P.*)' @classmethod def parse(cls, source): From dd59f8d724ccd3c3e86b201511ceb07da483a53e Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 1 Nov 2017 23:41:52 +1300 Subject: [PATCH 045/322] Greasemonkey: add more end2end tests Test document-end and noframes. Because coverage.py told me to. Hopefully this doesn't slow the test run down too much, particularly the "should not be logged" bit. I'm just reusing and existing test html page that used an iframe because I'm lazy. --- tests/end2end/features/javascript.feature | 20 ++++++++--- tests/end2end/features/test_javascript_bdd.py | 35 +++++++++++++------ 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature index 66d108125..aaad84be3 100644 --- a/tests/end2end/features/javascript.feature +++ b/tests/end2end/features/javascript.feature @@ -124,11 +124,23 @@ Feature: Javascript stuff And I run :tab-next Then the window sizes should be the same - Scenario: Have a greasemonkey script run on a page - When I have a greasemonkey file saved + Scenario: Have a greasemonkey script run at page start + When I have a greasemonkey file saved for document-start with noframes unset And I run :greasemonkey-reload - And I open data/title.html + And I open data/hints/iframe.html # This second reload is required in webengine < 5.8 for scripts # registered to run at document-start, some sort of timing issue. And I run :reload - Then the javascript message "Script is running." should be logged + Then the javascript message "Script is running on /data/hints/iframe.html" should be logged + + Scenario: Have a greasemonkey script running on frames + When I have a greasemonkey file saved for document-end with noframes unset + And I run :greasemonkey-reload + And I open data/hints/iframe.html + Then the javascript message "Script is running on /data/hints/html/wrapped.html" should be logged + + Scenario: Have a greasemonkey script running on noframes + When I have a greasemonkey file saved for document-end with noframes set + And I run :greasemonkey-reload + And I open data/hints/iframe.html + Then the javascript message "Script is running on /data/hints/html/wrapped.html" should not be logged diff --git a/tests/end2end/features/test_javascript_bdd.py b/tests/end2end/features/test_javascript_bdd.py index cb81e1ef6..16896d4b5 100644 --- a/tests/end2end/features/test_javascript_bdd.py +++ b/tests/end2end/features/test_javascript_bdd.py @@ -32,22 +32,37 @@ def check_window_sizes(quteproc): visible_size = visible.message.split()[-1] assert hidden_size == visible_size -test_gm_script=""" + +test_gm_script = r""" // ==UserScript== // @name Qutebrowser test userscript // @namespace invalid.org -// @include http://localhost:*/data/title.html +// @include http://localhost:*/data/hints/iframe.html +// @include http://localhost:*/data/hints/html/wrapped.html // @exclude ??? -// @run-at document-start +// @run-at {stage} +// {frames} // ==/UserScript== console.log("Script is running on " + window.location.pathname); """ -@bdd.when("I have a greasemonkey file saved") -def create_greasemonkey_file(quteproc): - script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey') - os.mkdir(script_path) - file_path = os.path.join(script_path, 'test.user.js') - with open(file_path, 'w', encoding='utf-8') as f: - f.write(test_gm_script) +@bdd.when(bdd.parsers.parse("I have a greasemonkey file saved for {stage} " + "with noframes {frameset}")) +def create_greasemonkey_file(quteproc, stage, frameset): + script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey') + try: + os.mkdir(script_path) + except FileExistsError: + pass + file_path = os.path.join(script_path, 'test.user.js') + if frameset == "set": + frames = "@noframes" + elif frameset == "unset": + frames = "" + else: + raise ValueError("noframes can only be set or unset, " + "not {}".format(frameset)) + with open(file_path, 'w', encoding='utf-8') as f: + f.write(test_gm_script.format(stage=stage, + frames=frames)) From 92b48e77c79cf5b1206fdd953a92678315f083e6 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 1 Nov 2017 23:45:31 +1300 Subject: [PATCH 046/322] Greasemonkey: add unit tests for GreasemonkeyManager --- tests/unit/javascript/test_greasemonkey.py | 108 +++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/unit/javascript/test_greasemonkey.py diff --git a/tests/unit/javascript/test_greasemonkey.py b/tests/unit/javascript/test_greasemonkey.py new file mode 100644 index 000000000..b0ba64bdf --- /dev/null +++ b/tests/unit/javascript/test_greasemonkey.py @@ -0,0 +1,108 @@ +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: +# Copyright 2017 Florian Bruhin (The Compiler) + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +"""Tests for qutebrowser.browser.greasemonkey.""" + +import os +import logging + +import pytest +from PyQt5.QtCore import QUrl + +from qutebrowser.browser import greasemonkey + +test_gm_script = """ +// ==UserScript== +// @name Qutebrowser test userscript +// @namespace invalid.org +// @include http://localhost:*/data/title.html +// @match http://trolol* +// @exclude https://badhost.xxx/* +// @run-at document-start +// ==/UserScript== +console.log("Script is running."); +""" + +pytestmark = pytest.mark.usefixtures('data_tmpdir') + + +def save_script(script_text, filename): + script_path = greasemonkey._scripts_dir() + try: + os.mkdir(script_path) + except FileExistsError: + pass + file_path = os.path.join(script_path, filename) + with open(file_path, 'w', encoding='utf-8') as f: + f.write(script_text) + + +def test_all(): + """Test that a script gets read from file, parsed and returned.""" + save_script(test_gm_script, 'test.user.js') + + gm_manager = greasemonkey.GreasemonkeyManager() + assert (gm_manager.all_scripts()[0].name == + "Qutebrowser test userscript") + + +@pytest.mark.parametrize("url, expected_matches", [ + # included + ('http://trololololololo.com/', 1), + # neither included nor excluded + ('http://aaaaaaaaaa.com/', 0), + # excluded + ('https://badhost.xxx/', 0), +]) +def test_get_scripts_by_url(url, expected_matches): + """Check greasemonkey include/exclude rules work.""" + save_script(test_gm_script, 'test.user.js') + gm_manager = greasemonkey.GreasemonkeyManager() + + scripts = gm_manager.scripts_for(QUrl(url)) + assert (len(scripts.start + scripts.end + scripts.idle) == + expected_matches) + + +def test_no_metadata(caplog): + """Run on all sites at document-end is the default.""" + save_script("var nothing = true;\n", 'nothing.user.js') + + with caplog.at_level(logging.WARNING): + gm_manager = greasemonkey.GreasemonkeyManager() + + scripts = gm_manager.scripts_for(QUrl('http://notamatch.invalid/')) + assert len(scripts.start + scripts.end + scripts.idle) == 1 + assert len(scripts.end) == 1 + + +def test_bad_scheme(caplog): + """qute:// isn't in the list of allowed schemes.""" + save_script("var nothing = true;\n", 'nothing.user.js') + + with caplog.at_level(logging.WARNING): + gm_manager = greasemonkey.GreasemonkeyManager() + + scripts = gm_manager.scripts_for(QUrl('qute://settings')) + assert len(scripts.start + scripts.end + scripts.idle) == 0 + + +def test_load_emits_signal(qtbot): + gm_manager = greasemonkey.GreasemonkeyManager() + with qtbot.wait_signal(gm_manager.scripts_reloaded): + gm_manager.load_scripts() From 8a5b42ffbd6ad878fa5810819086b1db77fe8842 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sat, 4 Nov 2017 16:11:58 +1300 Subject: [PATCH 047/322] Greasemonkey: es6ify the greasemonkey wrapper js. Because backwards compatibility sucks I guess. --- .../javascript/greasemonkey_wrapper.js | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index 7a271375d..e86991040 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -1,11 +1,11 @@ (function () { - var _qute_script_id = "__gm_"+{{ scriptName | tojson }}; + const _qute_script_id = "__gm_"+{{ scriptName | tojson }}; function GM_log(text) { console.log(text); } - var GM_info = { + const GM_info = { 'script': {{ scriptInfo }}, 'scriptMetaStr': {{ scriptMeta | tojson }}, 'scriptWillUpdate': false, @@ -15,7 +15,7 @@ function checkKey(key, funcName) { if (typeof key !== "string") { - throw new Error(funcName+" requires the first parameter to be of type string, not '"+typeof key+"'"); + throw new Error(`${funcName} requires the first parameter to be of type string, not '${typeof key}'`); } } @@ -24,7 +24,7 @@ if (typeof value !== "string" && typeof value !== "number" && typeof value !== "boolean") { - throw new Error("GM_setValue requires the second parameter to be of type string, number or boolean, not '"+typeof value+"'"); + throw new Error(`GM_setValue requires the second parameter to be of type string, number or boolean, not '${typeof value}'`); } localStorage.setItem(_qute_script_id + key, value); } @@ -40,8 +40,8 @@ } function GM_listValues() { - var keys = []; - for (var i = 0; i < localStorage.length; i++) { + let keys = []; + for (let i = 0; i < localStorage.length; i++) { if (localStorage.key(i).startsWith(_qute_script_id)) { keys.push(localStorage.key(i).slice(_qute_script_id.length)); } @@ -63,7 +63,7 @@ } // build XMLHttpRequest object - var oXhr = new XMLHttpRequest(); + let oXhr = new XMLHttpRequest(); // run it if ("onreadystatechange" in details) { oXhr.onreadystatechange = function () { @@ -80,7 +80,7 @@ oXhr.open(details.method, details.url, true); if ("headers" in details) { - for (var header in details.headers) { + for (let header in details.headers) { oXhr.setRequestHeader(header, details.headers[header]); } } @@ -93,11 +93,11 @@ } function GM_addStyle(/* String */ styles) { - var oStyle = document.createElement("style"); + let oStyle = document.createElement("style"); oStyle.setAttribute("type", "text/css"); oStyle.appendChild(document.createTextNode(styles)); - var head = document.getElementsByTagName("head")[0]; + let head = document.getElementsByTagName("head")[0]; if (head === undefined) { document.onreadystatechange = function () { if (document.readyState == "interactive") { @@ -110,7 +110,7 @@ } } - var unsafeWindow = window; + const unsafeWindow = window; //====== The actual user script source ======// {{ scriptSource }} From df624944f99a0dca33900ddaed2cd49f73880ac6 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 8 Nov 2017 21:03:58 +1300 Subject: [PATCH 048/322] Greasemonkey: webkit: injected all scripts on loadFinished. The signal we were using to inject greasemonkey scripts registered to run at document-start (javaScriptWindowObjectCleared) was unreliable to non-existant. The initialLayoutCompleted signal is a bit of an odd duck too I suppose. Anyway, we don't anticipate any scripts would break from being injected when the page is finished loaded that wouldn't already have been flaky due to the complexities of the modern web. If there is an issue hopefully someone raises an issue and we can look into it. --- qutebrowser/browser/webkit/webpage.py | 51 +++++++-------------------- 1 file changed, 13 insertions(+), 38 deletions(-) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 299815cc3..f9bd51194 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -86,33 +86,18 @@ class BrowserPage(QWebPage): self.on_save_frame_state_requested) self.restoreFrameStateRequested.connect( self.on_restore_frame_state_requested) - self.connect_userjs_signals(None) + self.connect_userjs_signals(self.mainFrame()) self.frameCreated.connect(self.connect_userjs_signals) @pyqtSlot('QWebFrame*') - def connect_userjs_signals(self, frame_arg): - """ - Connect the signals used as triggers for injecting user - javascripts into `frame_arg`. - """ - # If we pass whatever self.mainFrame() or self.currentFrame() returns - # at init time into the partial functions which the signals - # below call then the signals don't seem to be called at all for - # the main frame of the first tab. I have no idea why I am - # seeing this behavior. Replace the None in the call to this - # function in __init__ with self.mainFrame() and try for - # yourself. - if frame_arg: - frame = frame_arg - else: - frame = self.mainFrame() + def connect_userjs_signals(self, frame): + """Connect userjs related signals to `frame`. - frame.javaScriptWindowObjectCleared.connect( - functools.partial(self.inject_userjs, frame_arg, load='start')) - frame.initialLayoutCompleted.connect( - functools.partial(self.inject_userjs, frame_arg, load='end')) + Connect the signals used as triggers for injecting user + javascripts into the passed QWebFrame. + """ frame.loadFinished.connect( - functools.partial(self.inject_userjs, frame_arg, load='idle')) + functools.partial(self.inject_userjs, frame)) def javaScriptPrompt(self, frame, js_msg, default): """Override javaScriptPrompt to use qutebrowser prompts.""" @@ -311,32 +296,22 @@ class BrowserPage(QWebPage): self.error_occurred = False @pyqtSlot() - def inject_userjs(self, frame, load): + def inject_userjs(self, frame): """Inject user javascripts into the page. Args: - frame: The QWebFrame to inject the user scripts into, or - None for the main frame. - load: The page load stage to inject the corresponding - scripts for. Support values are "start", "end" and - "idle", corresponding to the allowed values of the - `@run-at` directive in the greasemonkey metadata spec. + frame: The QWebFrame to inject the user scripts into. """ - if not frame: - frame = self.mainFrame() url = frame.url() if url.isEmpty(): url = frame.requestedUrl() greasemonkey = objreg.get('greasemonkey') scripts = greasemonkey.scripts_for(url) - - if load == "start": - toload = scripts.start - elif load == "end": - toload = scripts.end - elif load == "idle": - toload = scripts.idle + # QtWebKit has trouble providing us with signals representing + # page load progress at reasonable times, so we just load all + # scripts on the same event. + toload = scripts.start + scripts.end + scripts.idle if url.isEmpty(): # This happens during normal usage like with view source but may From 6933bc05b4ba93ef6d8d72ec582362a7db36aef5 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 20 Nov 2017 07:31:01 +0100 Subject: [PATCH 049/322] Add some debug logging for GreaseMonkey with QtWebKit --- qutebrowser/browser/webkit/webpage.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index f9bd51194..7c67ab64b 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -96,6 +96,8 @@ class BrowserPage(QWebPage): Connect the signals used as triggers for injecting user javascripts into the passed QWebFrame. """ + log.greasemonkey.debug("Connecting to frame {} ({})" + .format(frame, frame.url().toDisplayString())) frame.loadFinished.connect( functools.partial(self.inject_userjs, frame)) @@ -306,6 +308,9 @@ class BrowserPage(QWebPage): if url.isEmpty(): url = frame.requestedUrl() + log.greasemonkey.debug("inject_userjs called for {} ({})" + .format(frame, url.toDisplayString())) + greasemonkey = objreg.get('greasemonkey') scripts = greasemonkey.scripts_for(url) # QtWebKit has trouble providing us with signals representing From db353c40300ef8d4066026b5650c27daa2da61e3 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 20 Nov 2017 07:31:13 +0100 Subject: [PATCH 050/322] Connect the page signal for GreaseMonkey Looks like we don't get the mainFrame's loadFinished signal properly. --- qutebrowser/browser/webkit/webpage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 7c67ab64b..24a213a42 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -86,7 +86,8 @@ class BrowserPage(QWebPage): self.on_save_frame_state_requested) self.restoreFrameStateRequested.connect( self.on_restore_frame_state_requested) - self.connect_userjs_signals(self.mainFrame()) + self.loadFinished.connect( + functools.partial(self.inject_userjs, self.mainFrame())) self.frameCreated.connect(self.connect_userjs_signals) @pyqtSlot('QWebFrame*') From 0e80be2d30ac90b82db968d077d498ad9c1e8f97 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 20 Nov 2017 08:31:10 +0100 Subject: [PATCH 051/322] Clear end2end test data again after initializing If we don't do this, earlier tests can affect later ones when e.g. using "... should not be logged", as we don't really wait until a test has been fully finished. --- tests/end2end/fixtures/quteprocess.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index 27c347ca4..3164f5fde 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -527,6 +527,7 @@ class QuteProc(testprocess.Process): super().before_test() self.send_cmd(':config-clear') self._init_settings() + self.clear_data() def _init_settings(self): """Adjust some qutebrowser settings after starting.""" From 0381d74e9aac2d045567db13ffc2ccca55d57160 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Mon, 27 Nov 2017 20:06:29 +1300 Subject: [PATCH 052/322] Greasemonkey: privatise some utility functions --- qutebrowser/browser/webkit/webpage.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 24a213a42..89b293869 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -87,11 +87,11 @@ class BrowserPage(QWebPage): self.restoreFrameStateRequested.connect( self.on_restore_frame_state_requested) self.loadFinished.connect( - functools.partial(self.inject_userjs, self.mainFrame())) - self.frameCreated.connect(self.connect_userjs_signals) + functools.partial(self._inject_userjs, self.mainFrame())) + self.frameCreated.connect(self._connect_userjs_signals) @pyqtSlot('QWebFrame*') - def connect_userjs_signals(self, frame): + def _connect_userjs_signals(self, frame): """Connect userjs related signals to `frame`. Connect the signals used as triggers for injecting user @@ -100,7 +100,7 @@ class BrowserPage(QWebPage): log.greasemonkey.debug("Connecting to frame {} ({})" .format(frame, frame.url().toDisplayString())) frame.loadFinished.connect( - functools.partial(self.inject_userjs, frame)) + functools.partial(self._inject_userjs, frame)) def javaScriptPrompt(self, frame, js_msg, default): """Override javaScriptPrompt to use qutebrowser prompts.""" @@ -298,8 +298,7 @@ class BrowserPage(QWebPage): else: self.error_occurred = False - @pyqtSlot() - def inject_userjs(self, frame): + def _inject_userjs(self, frame): """Inject user javascripts into the page. Args: @@ -309,7 +308,7 @@ class BrowserPage(QWebPage): if url.isEmpty(): url = frame.requestedUrl() - log.greasemonkey.debug("inject_userjs called for {} ({})" + log.greasemonkey.debug("_inject_userjs called for {} ({})" .format(frame, url.toDisplayString())) greasemonkey = objreg.get('greasemonkey') From d29cf1ee4d0cf68b417d77de617ae7fbe24e3814 Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Thu, 30 Nov 2017 00:09:28 +0100 Subject: [PATCH 053/322] lazy sessions, restore if visible, forward user after restore --- qutebrowser/html/back.html | 53 ++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index 83f5cb74e..94d16824e 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -1,16 +1,53 @@ {% extends "base.html" %} {% block script %} -setTimeout(function() { - /* drop first focus event, to avoid problems - (allow to go easily to newer history entries) */ - window.onfocus = function() { - window.onfocus = null; - window.history.back(); - }; -}, 1000); +const STATE_BACK = "back"; +const STATE_FORWARD = "forward"; + +function switch_state(new_state) { + history.replaceState( + new_state, + document.title, + location.pathname+location.hash); +} + +function go_back() { + switch_state(STATE_FORWARD); + history.back(); +} +function go_forward() { + switch_state(STATE_BACK); + history.forward(); +} + +// there are three states +// default: register focus listener, +// on focus: go back and switch to the state forward +// back: user came from a later history entry +// -> switch to the state forward, +// forward him to the previous history entry +// forward: user came from a previous history entry +// -> switch to the state back, +// forward him to the next history entry +switch (history.state) { + case STATE_BACK: + go_back(); + break; + case STATE_FORWARD: + go_forward(); + break; + default: + if (!document.hidden) { + go_back(); + break; + } + + document.addEventListener("visibilitychange", go_back); + break; +} {% endblock %} {% block content %} +

If you see this site something went wrong or you reached the end of the history or you disabled javascript.

{% endblock %} From f7b0ac503ec7339109b04abb5e03b4462a10f365 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Wed, 29 Nov 2017 02:29:53 -0600 Subject: [PATCH 054/322] generate pytest envs with tox factors This eliminates all separate pytest envs in favor of conditionals in [testenv]. This requires renaming some environments to make the lack of certain functionality explicit: - instead of omitting pyqt{version}, use pyqtlink to use host PyQt tox.ini: eliminate -nocov It is possible to set the `PYTEST_ADDOPTS` environment variable to enable coverage checks, rather than a new command. --- tox.ini | 142 ++++++++++++++------------------------------------------ 1 file changed, 34 insertions(+), 108 deletions(-) diff --git a/tox.ini b/tox.ini index c0395a621..f805e006b 100644 --- a/tox.ini +++ b/tox.ini @@ -13,120 +13,24 @@ skipsdist = true setenv = QT_QPA_PLATFORM_PLUGIN_PATH={envdir}/Lib/site-packages/PyQt5/plugins/platforms PYTEST_QT_API=pyqt5 + pyqt{,56,571,58,59}: QUTE_BDD_WEBENGINE=true + cov: PYTEST_ADDOPTS=--cov --cov-report xml --cov-report=html --cov-report= passenv = PYTHON DISPLAY XAUTHORITY HOME USERNAME USER CI TRAVIS XDG_* QUTE_* DOCKER +basepython = + py35: python3.5 + py36: python3.6 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-tests.txt + pyqt: PyQt5 + pyqt56: PyQt5==5.6 + pyqt571: PyQt5==5.7.1 + pyqt58: PyQt5==5.8.2 + pyqt59: PyQt5==5.9.2 commands = - {envpython} scripts/link_pyqt.py --tox {envdir} + pyqtlink: {envpython} scripts/link_pyqt.py --tox {envdir} {envpython} -bb -m pytest {posargs:tests} - -# test envs with PyQt5 from PyPI - -[testenv:py35-pyqt56] -basepython = python3.5 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.6 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py35-pyqt571] -basepython = python3.5 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.7.1 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py36-pyqt571] -basepython = python3.6 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.7.1 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py35-pyqt58] -basepython = python3.5 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.8.2 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py36-pyqt58] -basepython = {env:PYTHON:python3.6} -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.8.2 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py35-pyqt59] -basepython = python3.5 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.9.2 -commands = {envpython} -bb -m pytest {posargs:tests} - -[testenv:py36-pyqt59] -basepython = {env:PYTHON:python3.6} -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.9.2 -commands = {envpython} -bb -m pytest {posargs:tests} - -# test envs with coverage - -[testenv:py35-pyqt59-cov] -basepython = python3.5 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.9.2 -commands = - {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests} - {envpython} scripts/dev/check_coverage.py {posargs} - -[testenv:py36-pyqt59-cov] -basepython = python3.6 -setenv = - {[testenv]setenv} - QUTE_BDD_WEBENGINE=true -passenv = {[testenv]passenv} -deps = - {[testenv]deps} - PyQt5==5.9.2 -commands = - {envpython} -bb -m pytest --cov --cov-report xml --cov-report=html --cov-report= {posargs:tests} - {envpython} scripts/dev/check_coverage.py {posargs} + cov: {envpython} scripts/dev/check_coverage.py {posargs} # other envs @@ -196,6 +100,16 @@ setenv = PYTHONPATH={toxinidir} commands = {envpython} scripts/dev/run_vulture.py +[testenv:vulture-pyqtlink] +basepython = python3 +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/misc/requirements/requirements-vulture.txt +setenv = PYTHONPATH={toxinidir} +commands = + {envpython} scripts/link_pyqt.py --tox {envdir} + {envpython} scripts/dev/run_vulture.py + [testenv:pylint] basepython = {env:PYTHON:python3} ignore_errors = true @@ -208,6 +122,18 @@ commands = {envpython} -m pylint scripts qutebrowser --output-format=colorized --reports=no {posargs} {envpython} scripts/dev/run_pylint_on_tests.py {toxinidir} --output-format=colorized --reports=no {posargs} +[testenv:pylint-pyqtlink] +basepython = {env:PYTHON:python3} +ignore_errors = true +passenv = +deps = + {[testenv]deps} + -r{toxinidir}/misc/requirements/requirements-pylint.txt +commands = + {envpython} scripts/link_pyqt.py --tox {envdir} + {envpython} -m pylint scripts qutebrowser --output-format=colorized --reports=no {posargs} + {envpython} scripts/dev/run_pylint_on_tests.py {toxinidir} --output-format=colorized --reports=no {posargs} + [testenv:pylint-master] basepython = python3 passenv = {[testenv:pylint]passenv} From 2f231c86ac76cd47038759331b3a1b26418043e0 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Thu, 30 Nov 2017 08:35:02 -0600 Subject: [PATCH 055/322] update tox env name in CI config Use py36-pyqtlink instead of py36 for macOS --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 65d917d73..e92df0b6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ matrix: - os: linux env: TESTENV=py36-pyqt59-cov - os: osx - env: TESTENV=py36 OSX=sierra + env: TESTENV=py36-pyqtlink OSX=sierra osx_image: xcode8.3 language: generic # https://github.com/qutebrowser/qutebrowser/issues/2013 From b58cfead05b201c3664fde8657556c5b260a9e6a Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Thu, 30 Nov 2017 16:05:01 +0100 Subject: [PATCH 056/322] style fixed --- qutebrowser/browser/qutescheme.py | 4 ++-- qutebrowser/html/back.html | 5 +++-- qutebrowser/misc/sessions.py | 10 +++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 9771f6db1..247e12a78 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -433,8 +433,8 @@ def qute_back(url): Simple page to free ram / lazy load a site, goes back on focusing the tab. """ html = jinja.render( - 'back.html', - title='Suspended: ' + urllib.parse.unquote(url.fragment())) + 'back.html', + title='Suspended: ' + urllib.parse.unquote(url.fragment())) return 'text/html', html diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index 94d16824e..b6190feda 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -8,13 +8,14 @@ function switch_state(new_state) { history.replaceState( new_state, document.title, - location.pathname+location.hash); + location.pathname + location.hash); } function go_back() { switch_state(STATE_FORWARD); history.back(); } + function go_forward() { switch_state(STATE_BACK); history.forward(); @@ -49,5 +50,5 @@ switch (history.state) { {% block content %} -

If you see this site something went wrong or you reached the end of the history or you disabled javascript.

+

If you see this site something went wrong, or you reached the end of the history, or you disabled javascript.

{% endblock %} diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 1b0106f1b..237e46189 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -336,11 +336,11 @@ class SessionManager(QObject): # -> dropwhile empty if not session.lazy_session lazy_index = len(data['history']) gen = itertools.chain( - itertools.takewhile(lambda _: not lazy_load, - enumerate(data['history'])), - enumerate(lazy_load), - itertools.dropwhile(lambda i: i[0] < lazy_index, - enumerate(data['history']))) + itertools.takewhile(lambda _: not lazy_load, + enumerate(data['history'])), + enumerate(lazy_load), + itertools.dropwhile(lambda i: i[0] < lazy_index, + enumerate(data['history']))) for i, histentry in gen: user_data = {} From a5d0b9851ace4a3a335748925cc971b7f7407def Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Thu, 30 Nov 2017 14:14:11 -0600 Subject: [PATCH 057/322] tox.ini: remove pyqt5.6, use requirements-pyqt.txt --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index f805e006b..1c87d1a6c 100644 --- a/tox.ini +++ b/tox.ini @@ -22,8 +22,7 @@ basepython = deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-tests.txt - pyqt: PyQt5 - pyqt56: PyQt5==5.6 + pyqt: -r{toxinidir}/misc/requirements/requirements-pyqt.txt pyqt571: PyQt5==5.7.1 pyqt58: PyQt5==5.8.2 pyqt59: PyQt5==5.9.2 From 6b762037809b9057774bb88f333466d6d1ba9bd3 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Thu, 30 Nov 2017 14:21:37 -0600 Subject: [PATCH 058/322] update contributing.asciidoc with -pyqtlink envs --- doc/contributing.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing.asciidoc b/doc/contributing.asciidoc index afbb752c5..6f0b1bb72 100644 --- a/doc/contributing.asciidoc +++ b/doc/contributing.asciidoc @@ -97,7 +97,7 @@ unittests and several linters/checkers. Currently, the following tox environments are available: * Tests using https://www.pytest.org[pytest]: - - `py35`, `py36`: Run pytest for python 3.5/3.6 with the system-wide PyQt. + - `py35-pyqtlink`, `py36-pyqtlink`: Run pytest for python 3.5/3.6 with the system-wide PyQt. - `py36-pyqt57`, ..., `py36-pyqt59`: Run pytest with the given PyQt version (`py35-*` also works). - `py36-pyqt59-cov`: Run with coverage support (other Python/PyQt versions work too). * `flake8`: Run various linting checks via https://pypi.python.org/pypi/flake8[flake8]. From 49485ca220543ce9fbcc48da9f3b69feef695822 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Thu, 30 Nov 2017 16:58:14 -0600 Subject: [PATCH 059/322] tox.ini: fix conditional syntax errors `{[testenv]deps}` was passing conditionals in their raw form; this simply lists them manually to avoid this. --- tox.ini | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 1c87d1a6c..de41f2e0f 100644 --- a/tox.ini +++ b/tox.ini @@ -107,14 +107,15 @@ deps = setenv = PYTHONPATH={toxinidir} commands = {envpython} scripts/link_pyqt.py --tox {envdir} - {envpython} scripts/dev/run_vulture.py + {[testenv:vulture]commands} [testenv:pylint] basepython = {env:PYTHON:python3} ignore_errors = true passenv = deps = - {[testenv]deps} + -r{toxinidir}/requirements.txt + -r{toxinidir}/misc/requirements/requirements-tests.txt -r{toxinidir}/misc/requirements/requirements-pylint.txt -r{toxinidir}/misc/requirements/requirements-pyqt.txt commands = @@ -126,18 +127,19 @@ basepython = {env:PYTHON:python3} ignore_errors = true passenv = deps = - {[testenv]deps} + -r{toxinidir}/requirements.txt + -r{toxinidir}/misc/requirements/requirements-tests.txt -r{toxinidir}/misc/requirements/requirements-pylint.txt commands = {envpython} scripts/link_pyqt.py --tox {envdir} - {envpython} -m pylint scripts qutebrowser --output-format=colorized --reports=no {posargs} - {envpython} scripts/dev/run_pylint_on_tests.py {toxinidir} --output-format=colorized --reports=no {posargs} + {[testenv:pylint]commands} [testenv:pylint-master] basepython = python3 passenv = {[testenv:pylint]passenv} deps = - {[testenv]deps} + -r{toxinidir}/requirements.txt + -r{toxinidir}/misc/requirements/requirements-tests.txt -r{toxinidir}/misc/requirements/requirements-pylint-master.txt commands = {envpython} scripts/link_pyqt.py --tox {envdir} From 780ac3f4c2bb5bd53d63531215f28dd07613d271 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Fri, 1 Dec 2017 11:34:47 -0500 Subject: [PATCH 060/322] Remove needles quteproc/server fixture deps. A few step definitions listed these in the parameters although they were unused. --- tests/end2end/features/test_editor_bdd.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/end2end/features/test_editor_bdd.py b/tests/end2end/features/test_editor_bdd.py index 983ded7ba..ec43c8477 100644 --- a/tests/end2end/features/test_editor_bdd.py +++ b/tests/end2end/features/test_editor_bdd.py @@ -49,7 +49,7 @@ def set_up_editor_replacement(quteproc, server, tmpdir, text, replacement): @bdd.when(bdd.parsers.parse('I set up a fake editor returning "{text}"')) -def set_up_editor(quteproc, server, tmpdir, text): +def set_up_editor(quteproc, tmpdir, text): """Set up editor.command to a small python script inserting a text.""" script = tmpdir / 'script.py' script.write(textwrap.dedent(""" @@ -63,13 +63,13 @@ def set_up_editor(quteproc, server, tmpdir, text): @bdd.when(bdd.parsers.parse('I set up a fake editor returning empty text')) -def set_up_editor_empty(quteproc, server, tmpdir): +def set_up_editor_empty(quteproc, tmpdir): """Set up editor.command to a small python script inserting empty text.""" - set_up_editor(quteproc, server, tmpdir, "") + set_up_editor(quteproc, tmpdir, "") @bdd.when(bdd.parsers.parse('I set up a fake editor that waits')) -def set_up_editor_wait(quteproc, server, tmpdir): +def set_up_editor_wait(quteproc, tmpdir): """Set up editor.command to a small python script inserting a text.""" pidfile = tmpdir / 'editor_pid' script = tmpdir / 'script.py' @@ -90,7 +90,7 @@ def set_up_editor_wait(quteproc, server, tmpdir): @bdd.when(bdd.parsers.parse('I kill the waiting editor')) -def kill_editor_wait(quteproc, server, tmpdir): +def kill_editor_wait(tmpdir): """Kill the waiting editor.""" pidfile = tmpdir / 'editor_pid' pid = int(pidfile.read()) From df6ff55b7a34d543b9235bf71f0131b2598ff1c9 Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Fri, 1 Dec 2017 10:51:41 -0600 Subject: [PATCH 061/322] allow pytest to default to link_pyqt link_pyqt now checks for LINK_PYQT_SKIP, allowing pytest env names like `py36` to work properly without negative conditionals in tox.ini --- scripts/link_pyqt.py | 4 ++++ tox.ini | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/link_pyqt.py b/scripts/link_pyqt.py index 57eeb9138..82f8cbac5 100644 --- a/scripts/link_pyqt.py +++ b/scripts/link_pyqt.py @@ -205,6 +205,10 @@ def main(): args = parser.parse_args() if args.tox: + #workaround for the lack of negative factors in tox.ini + if 'LINK_PYQT_SKIP' in os.environ: + print('LINK_PYQT_SKIP set, exiting...') + sys.exit(0) executable = get_tox_syspython(args.path) else: executable = sys.executable diff --git a/tox.ini b/tox.ini index de41f2e0f..2ada5ab58 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ skipsdist = true setenv = QT_QPA_PLATFORM_PLUGIN_PATH={envdir}/Lib/site-packages/PyQt5/plugins/platforms PYTEST_QT_API=pyqt5 + pyqt{,56,571,58,59}: LINK_PYQT_SKIP=true pyqt{,56,571,58,59}: QUTE_BDD_WEBENGINE=true cov: PYTEST_ADDOPTS=--cov --cov-report xml --cov-report=html --cov-report= passenv = PYTHON DISPLAY XAUTHORITY HOME USERNAME USER CI TRAVIS XDG_* QUTE_* DOCKER @@ -27,7 +28,7 @@ deps = pyqt58: PyQt5==5.8.2 pyqt59: PyQt5==5.9.2 commands = - pyqtlink: {envpython} scripts/link_pyqt.py --tox {envdir} + {envpython} scripts/link_pyqt.py --tox {envdir} {envpython} -bb -m pytest {posargs:tests} cov: {envpython} scripts/dev/check_coverage.py {posargs} From 5607cc2be8c379c002dc257b3fd9a0d606ede98d Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Fri, 1 Dec 2017 10:52:58 -0600 Subject: [PATCH 062/322] Revert "update contributing.asciidoc with -pyqtlink envs" This reverts commit 6b762037809b9057774bb88f333466d6d1ba9bd3. --- doc/contributing.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing.asciidoc b/doc/contributing.asciidoc index 6f0b1bb72..afbb752c5 100644 --- a/doc/contributing.asciidoc +++ b/doc/contributing.asciidoc @@ -97,7 +97,7 @@ unittests and several linters/checkers. Currently, the following tox environments are available: * Tests using https://www.pytest.org[pytest]: - - `py35-pyqtlink`, `py36-pyqtlink`: Run pytest for python 3.5/3.6 with the system-wide PyQt. + - `py35`, `py36`: Run pytest for python 3.5/3.6 with the system-wide PyQt. - `py36-pyqt57`, ..., `py36-pyqt59`: Run pytest with the given PyQt version (`py35-*` also works). - `py36-pyqt59-cov`: Run with coverage support (other Python/PyQt versions work too). * `flake8`: Run various linting checks via https://pypi.python.org/pypi/flake8[flake8]. From fbd325f8d1a95507b656b3111b11d87d3e20aa1b Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Fri, 1 Dec 2017 10:55:08 -0600 Subject: [PATCH 063/322] Revert "update tox env name in CI config" This reverts commit 2f231c86ac76cd47038759331b3a1b26418043e0. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e92df0b6b..65d917d73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ matrix: - os: linux env: TESTENV=py36-pyqt59-cov - os: osx - env: TESTENV=py36-pyqtlink OSX=sierra + env: TESTENV=py36 OSX=sierra osx_image: xcode8.3 language: generic # https://github.com/qutebrowser/qutebrowser/issues/2013 From b91a39be220c86fa2659e6cfa73b16c54a8686f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sat, 2 Dec 2017 19:22:14 +0100 Subject: [PATCH 064/322] Run shellcheck on Travis CI --- .travis.yml | 3 +++ scripts/dev/ci/travis_run.sh | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index 65d917d73..37387b620 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,9 @@ matrix: language: node_js python: null node_js: "lts/*" + - os: linux + env: TESTENV=shellcheck + services: docker fast_finish: true cache: diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index 2a5424fb9..70c41b0c3 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -14,6 +14,16 @@ elif [[ $TESTENV == eslint ]]; then # travis env cd qutebrowser/javascript || exit 1 eslint --color --report-unused-disable-directives . +elif [[ $TESTENV == shellcheck ]]; then + dev_scripts=$( find scripts/dev/ -name '*.sh' -print0 | xargs -0 ) + # false positive: we are using 'find -exec +' + # shellcheck disable=SC2038 + userscripts=$( find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + | xargs ) + IFS=" " read -r -a scripts <<< "$dev_scripts $userscripts" + docker run \ + -v "$PWD:/outside" \ + -w /outside \ + koalaman/shellcheck:stable "${scripts[@]}" else args=() [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb') From 595a53ad3b29570bed35764b0b2cbfdda9af2dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sat, 2 Dec 2017 19:23:55 +0100 Subject: [PATCH 065/322] Apply patch from #1697 --- misc/userscripts/open_download | 3 ++- misc/userscripts/password_fill | 6 +++--- misc/userscripts/qutedmenu | 12 +++--------- misc/userscripts/taskadd | 6 +++--- scripts/dev/ci/travis_install.sh | 6 +++--- scripts/dev/quit_segfault_test.sh | 10 ++++------ 6 files changed, 18 insertions(+), 25 deletions(-) diff --git a/misc/userscripts/open_download b/misc/userscripts/open_download index 6c1213b65..85ea7f849 100755 --- a/misc/userscripts/open_download +++ b/misc/userscripts/open_download @@ -76,6 +76,7 @@ crop-first-column() { ls-files() { # add the slash at the end of the download dir enforces to follow the # symlink, if the DOWNLOAD_DIR itself is a symlink + # shellcheck disable=SC2010 ls -Q --quoting-style escape -h -o -1 -A -t "${DOWNLOAD_DIR}/" \ | grep '^[-]' \ | cut -d' ' -f3- \ @@ -94,7 +95,7 @@ fi line=$(printf "%s\n" "${entries[@]}" \ | crop-first-column 55 \ | column -s $'\t' -t \ - | $ROFI_CMD "${rofi_default_args[@]}" $ROFI_ARGS) || true + | $ROFI_CMD "${rofi_default_args[@]}" "$ROFI_ARGS") || true if [ -z "$line" ]; then exit 0 fi diff --git a/misc/userscripts/password_fill b/misc/userscripts/password_fill index af394ac2c..f2157f190 100755 --- a/misc/userscripts/password_fill +++ b/misc/userscripts/password_fill @@ -236,7 +236,7 @@ pass_backend() { if ((match_line)) ; then # add entries with matching URL-tag while read -r -d "" passfile ; do - if $GPG "${GPG_OPTS}" -d "$passfile" \ + if $GPG "${GPG_OPTS[@]}" -d "$passfile" \ | grep --max-count=1 -iE "${match_line_pattern}${url}" > /dev/null then passfile="${passfile#$PREFIX}" @@ -269,7 +269,7 @@ pass_backend() { break fi fi - done < <($GPG "${GPG_OPTS}" -d "$path" ) + done < <($GPG "${GPG_OPTS[@]}" -d "$path" ) } } # ======================================================= @@ -283,7 +283,7 @@ secret_backend() { query_entries() { local domain="$1" while read -r line ; do - if [[ "$line" =~ "attribute.username = " ]] ; then + if [[ "$line" == "attribute.username ="* ]] ; then files+=("$domain ${line#${BASH_REMATCH[0]}}") fi done < <( secret-tool search --unlock --all domain "$domain" 2>&1 ) diff --git a/misc/userscripts/qutedmenu b/misc/userscripts/qutedmenu index 3f8b13514..b753c6d5b 100755 --- a/misc/userscripts/qutedmenu +++ b/misc/userscripts/qutedmenu @@ -35,17 +35,11 @@ get_selection() { # Main # https://github.com/halfwit/dotfiles/blob/master/.config/dmenu/font -if [[ -s $confdir/dmenu/font ]]; then - read -r font < "$confdir"/dmenu/font -fi +[[ -s $confdir/dmenu/font ]] && read -r font < "$confdir"/dmenu/font -if [[ $font ]]; then - opts+=(-fn "$font") -fi +[[ $font ]] && opts+=(-fn "$font") -if [[ -s $optsfile ]]; then - source "$optsfile" -fi +[[ -s $optsfile ]] && source "$optsfile" url=$(get_selection) url=${url/*http/http} diff --git a/misc/userscripts/taskadd b/misc/userscripts/taskadd index 6add71c68..9c70fb978 100755 --- a/misc/userscripts/taskadd +++ b/misc/userscripts/taskadd @@ -25,12 +25,12 @@ [[ $QUTE_MODE == 'hints' ]] && title=$QUTE_SELECTED_TEXT || title=$QUTE_TITLE # try to add the task and grab the output -msg="$(task add $title $@ 2>&1)" +msg="$(task add "$title" "$*" 2>&1)" if [[ $? == 0 ]]; then # annotate the new task with the url, send the output back to the browser task +LATEST annotate "$QUTE_URL" - echo "message-info '$msg'" >> $QUTE_FIFO + echo "message-info '$msg'" >> "$QUTE_FIFO" else - echo "message-error '$msg'" >> $QUTE_FIFO + echo "message-error '$msg'" >> "$QUTE_FIFO" fi diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index 4c599aac6..45bb5fe1e 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -27,17 +27,17 @@ travis_retry() { local count=1 while (( count < 3 )); do if (( result != 0 )); then - echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2 + echo -e "\n${ANSI_RED}The command \"$*\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2 fi "$@" result=$? (( result == 0 )) && break - count=$(($count + 1)) + count=$(( count + 1 )) sleep 1 done if (( count > 3 )); then - echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2 + echo -e "\n${ANSI_RED}The command \"$*\" failed 3 times.${ANSI_RESET}\n" >&2 fi return $result diff --git a/scripts/dev/quit_segfault_test.sh b/scripts/dev/quit_segfault_test.sh index 655eb262a..389f125b9 100755 --- a/scripts/dev/quit_segfault_test.sh +++ b/scripts/dev/quit_segfault_test.sh @@ -1,14 +1,12 @@ -#!/bin/bash +#!/usr/bin/env bash -if [[ $PWD == */scripts ]]; then - cd .. -fi +[[ $PWD == */scripts ]] && cd .. echo > crash.log while :; do exit=0 - while (( $exit == 0)); do - duration=$(($RANDOM%10000)) + while (( exit == 0 )); do + duration=$(( RANDOM % 10000 )) python3 -m qutebrowser --debug ":later $duration quit" http://www.heise.de/ exit=$? done From 31710b7045923dc984035984565f23b76a4d5ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sat, 2 Dec 2017 19:31:52 +0100 Subject: [PATCH 066/322] Trivial fixes for shellcheck warnings --- misc/userscripts/cast | 4 ++-- misc/userscripts/dmenu_qutebrowser | 2 +- misc/userscripts/format_json | 4 ++-- misc/userscripts/open_download | 2 +- misc/userscripts/password_fill | 5 +++-- misc/userscripts/qutedmenu | 1 + misc/userscripts/rss | 10 +++++----- scripts/dev/ci/travis_backtrace.sh | 2 +- scripts/dev/ci/travis_install.sh | 8 ++++---- scripts/dev/download_release.sh | 6 +++--- 10 files changed, 23 insertions(+), 21 deletions(-) diff --git a/misc/userscripts/cast b/misc/userscripts/cast index da68297d8..f7b64df70 100755 --- a/misc/userscripts/cast +++ b/misc/userscripts/cast @@ -144,7 +144,7 @@ fi pkill -f "${program_}" # start youtube download in stream mode (-o -) into temporary file -youtube-dl -qo - "$1" > ${file_to_cast} & +youtube-dl -qo - "$1" > "${file_to_cast}" & ytdl_pid=$! msg info "Casting $1" >> "$QUTE_FIFO" @@ -153,4 +153,4 @@ tail -F "${file_to_cast}" | ${program_} - # cleanup remaining background process and file on disk kill ${ytdl_pid} -rm -rf ${tmpdir} +rm -rf "${tmpdir}" diff --git a/misc/userscripts/dmenu_qutebrowser b/misc/userscripts/dmenu_qutebrowser index 9c809d5ad..82e6d2f18 100755 --- a/misc/userscripts/dmenu_qutebrowser +++ b/misc/userscripts/dmenu_qutebrowser @@ -41,7 +41,7 @@ [ -z "$QUTE_URL" ] && QUTE_URL='http://google.com' url=$(echo "$QUTE_URL" | cat - "$QUTE_CONFIG_DIR/quickmarks" "$QUTE_DATA_DIR/history" | dmenu -l 15 -p qutebrowser) -url=$(echo "$url" | sed -E 's/[^ ]+ +//g' | egrep "https?:" || echo "$url") +url=$(echo "$url" | sed -E 's/[^ ]+ +//g' | grep -E "https?:" || echo "$url") [ -z "${url// }" ] && exit diff --git a/misc/userscripts/format_json b/misc/userscripts/format_json index f756850f1..42d8dea14 100755 --- a/misc/userscripts/format_json +++ b/misc/userscripts/format_json @@ -35,12 +35,12 @@ FILE_SIZE=$(ls -s --block-size=1048576 "$QUTE_TEXT" | cut -d' ' -f1) # use pygments to pretty-up the json (syntax highlight) if file is less than 10MB if [ "$FILE_SIZE" -lt "10" ]; then - FORMATTED_JSON="$(echo "$FORMATTED_JSON" | pygmentize -l json -f html -O full,style=$STYLE)" + FORMATTED_JSON="$(echo "$FORMATTED_JSON" | pygmentize -l json -f html -O full,style="$STYLE")" fi # create a temp file and write the formatted json to that file TEMP_FILE="$(mktemp --suffix '.html')" -echo "$FORMATTED_JSON" > $TEMP_FILE +echo "$FORMATTED_JSON" > "$TEMP_FILE" # send the command to qutebrowser to open the new file containing the formatted json diff --git a/misc/userscripts/open_download b/misc/userscripts/open_download index 85ea7f849..ecc1d7209 100755 --- a/misc/userscripts/open_download +++ b/misc/userscripts/open_download @@ -92,7 +92,7 @@ if [ "${#entries[@]}" -eq 0 ] ; then die "Download directory »${DOWNLOAD_DIR}« empty" fi -line=$(printf "%s\n" "${entries[@]}" \ +line=$(printf '%s\n' "${entries[@]}" \ | crop-first-column 55 \ | column -s $'\t' -t \ | $ROFI_CMD "${rofi_default_args[@]}" "$ROFI_ARGS") || true diff --git a/misc/userscripts/password_fill b/misc/userscripts/password_fill index f2157f190..22497d987 100755 --- a/misc/userscripts/password_fill +++ b/misc/userscripts/password_fill @@ -178,7 +178,7 @@ choose_entry_menu() { if [ "$nr" -eq 1 ] && ! ((menu_if_one_entry)) ; then file="${files[0]}" else - file=$( printf "%s\n" "${files[@]}" | "${MENU_COMMAND[@]}" ) + file=$( printf '%s\n' "${files[@]}" | "${MENU_COMMAND[@]}" ) fi } @@ -303,6 +303,7 @@ pass_backend QUTE_CONFIG_DIR=${QUTE_CONFIG_DIR:-${XDG_CONFIG_HOME:-$HOME/.config}/qutebrowser/} PWFILL_CONFIG=${PWFILL_CONFIG:-${QUTE_CONFIG_DIR}/password_fill_rc} if [ -f "$PWFILL_CONFIG" ] ; then + # shellcheck source=/dev/null source "$PWFILL_CONFIG" fi init @@ -311,7 +312,7 @@ simplify_url "$QUTE_URL" query_entries "${simple_url}" no_entries_found # remove duplicates -mapfile -t files < <(printf "%s\n" "${files[@]}" | sort | uniq ) +mapfile -t files < <(printf '%s\n' "${files[@]}" | sort | uniq ) choose_entry if [ -z "$file" ] ; then # choose_entry didn't want any of these entries diff --git a/misc/userscripts/qutedmenu b/misc/userscripts/qutedmenu index b753c6d5b..de1b8d641 100755 --- a/misc/userscripts/qutedmenu +++ b/misc/userscripts/qutedmenu @@ -39,6 +39,7 @@ get_selection() { [[ $font ]] && opts+=(-fn "$font") +# shellcheck source=/dev/null [[ -s $optsfile ]] && source "$optsfile" url=$(get_selection) diff --git a/misc/userscripts/rss b/misc/userscripts/rss index 222d990a2..6680a259e 100755 --- a/misc/userscripts/rss +++ b/misc/userscripts/rss @@ -32,7 +32,7 @@ add_feed () { if grep -Fq "$1" "feeds"; then notice "$1 is saved already." else - printf "%s\n" "$1" >> "feeds" + printf '%s\n' "$1" >> "feeds" fi } @@ -57,7 +57,7 @@ notice () { # Update a database of a feed and open new URLs read_items () { - cd read_urls + cd read_urls || return feed_file="$(echo "$1" | tr -d /)" feed_temp_file="$(mktemp "$feed_file.tmp.XXXXXXXXXX")" feed_new_items="$(mktemp "$feed_file.new.XXXXXXXXXX")" @@ -75,7 +75,7 @@ read_items () { cat "$feed_new_items" >> "$feed_file" sort -o "$feed_file" "$feed_file" rm "$feed_temp_file" "$feed_new_items" - fi | while read item; do + fi | while read -r item; do echo "open -t $item" > "$QUTE_FIFO" done } @@ -85,7 +85,7 @@ if [ ! -d "$config_dir/read_urls" ]; then mkdir -p "$config_dir/read_urls" fi -cd "$config_dir" +cd "$config_dir" || exit if [ $# != 0 ]; then for arg in "$@"; do @@ -115,7 +115,7 @@ if < /dev/null grep --help 2>&1 | grep -q -- -a; then text_only="-a" fi -while read feed_url; do +while read -r feed_url; do read_items "$feed_url" & done < "$config_dir/feeds" diff --git a/scripts/dev/ci/travis_backtrace.sh b/scripts/dev/ci/travis_backtrace.sh index c94d1ff06..4027f7c10 100644 --- a/scripts/dev/ci/travis_backtrace.sh +++ b/scripts/dev/ci/travis_backtrace.sh @@ -6,7 +6,7 @@ case $TESTENV in py3*-pyqt*) - exe=$(readlink -f .tox/$TESTENV/bin/python) + exe=$(readlink -f ".tox/$TESTENV/bin/python") full= ;; *) diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index 45bb5fe1e..18ac44048 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -21,13 +21,13 @@ # Stolen from https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/templates/header.sh # and adjusted to use ((...)) travis_retry() { - local ANSI_RED="\033[31;1m" - local ANSI_RESET="\033[0m" + local ANSI_RED='\033[31;1m' + local ANSI_RESET='\033[0m' local result=0 local count=1 while (( count < 3 )); do if (( result != 0 )); then - echo -e "\n${ANSI_RED}The command \"$*\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2 + echo -e "\\n${ANSI_RED}The command \"$*\" failed. Retrying, $count of 3.${ANSI_RESET}\\n" >&2 fi "$@" result=$? @@ -37,7 +37,7 @@ travis_retry() { done if (( count > 3 )); then - echo -e "\n${ANSI_RED}The command \"$*\" failed 3 times.${ANSI_RESET}\n" >&2 + echo -e "\\n${ANSI_RED}The command \"$*\" failed 3 times.${ANSI_RESET}\\n" >&2 fi return $result diff --git a/scripts/dev/download_release.sh b/scripts/dev/download_release.sh index 7ec4d9159..66242ec8e 100644 --- a/scripts/dev/download_release.sh +++ b/scripts/dev/download_release.sh @@ -11,7 +11,7 @@ if [[ $# != 1 ]]; then exit 1 fi -cd "$tmpdir" +cd "$tmpdir" || exit mkdir windows base="https://github.com/qutebrowser/qutebrowser/releases/download/v$1" @@ -21,13 +21,13 @@ wget "$base/qutebrowser-$1.tar.gz.asc" || exit 1 wget "$base/qutebrowser-$1.dmg" || exit 1 wget "$base/qutebrowser_${1}-1_all.deb" || exit 1 -cd windows +cd windows || exit wget "$base/qutebrowser-${1}-amd64.msi" || exit 1 wget "$base/qutebrowser-${1}-win32.msi" || exit 1 wget "$base/qutebrowser-${1}-windows-standalone-amd64.zip" || exit 1 wget "$base/qutebrowser-${1}-windows-standalone-win32.zip" || exit 1 dest="/srv/http/qutebrowser/releases/v$1" -cd "$oldpwd" +cd "$oldpwd" || exit sudo mv "$tmpdir" "$dest" sudo chown -R http:http "$dest" From dd589f180bdd97670673957c43d4dfccd2ae6054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sat, 2 Dec 2017 19:38:02 +0100 Subject: [PATCH 067/322] Fix remaining shellcheck warnings --- misc/userscripts/format_json | 7 +++---- misc/userscripts/password_fill | 2 +- misc/userscripts/taskadd | 4 +--- misc/userscripts/view_in_mpv | 2 +- scripts/dev/ci/travis_backtrace.sh | 2 +- scripts/dev/download_release.sh | 23 ++++++++++++----------- 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/misc/userscripts/format_json b/misc/userscripts/format_json index 42d8dea14..10ff23e53 100755 --- a/misc/userscripts/format_json +++ b/misc/userscripts/format_json @@ -22,15 +22,14 @@ # default style to monokai if none is provided STYLE=${1:-monokai} # format json using jq -FORMATTED_JSON="$(cat "$QUTE_TEXT" | jq '.')" - -# if jq command failed or formatted json is empty, assume failure and terminate -if [ $? -ne 0 ] || [ -z "$FORMATTED_JSON" ]; then +if ! FORMATTED_JSON="$(jq . "$QUTE_TEXT")" || [ -z "$FORMATTED_JSON" ]; then echo "Invalid json, aborting..." exit 1 fi # calculate the filesize of the json document +# parsing the output of ls should be fine in this case since we set the block size +# shellcheck disable=SC2012 FILE_SIZE=$(ls -s --block-size=1048576 "$QUTE_TEXT" | cut -d' ' -f1) # use pygments to pretty-up the json (syntax highlight) if file is less than 10MB diff --git a/misc/userscripts/password_fill b/misc/userscripts/password_fill index 22497d987..5f30a6bf6 100755 --- a/misc/userscripts/password_fill +++ b/misc/userscripts/password_fill @@ -64,7 +64,7 @@ die() { javascript_escape() { # print the first argument in an escaped way, such that it can safely # be used within javascripts double quotes - sed "s,[\\\'\"],\\\&,g" <<< "$1" + sed "s,[\\\\'\"],\\\\&,g" <<< "$1" } # ======================================================= # diff --git a/misc/userscripts/taskadd b/misc/userscripts/taskadd index 9c70fb978..b1ded245c 100755 --- a/misc/userscripts/taskadd +++ b/misc/userscripts/taskadd @@ -25,9 +25,7 @@ [[ $QUTE_MODE == 'hints' ]] && title=$QUTE_SELECTED_TEXT || title=$QUTE_TITLE # try to add the task and grab the output -msg="$(task add "$title" "$*" 2>&1)" - -if [[ $? == 0 ]]; then +if msg="$(task add "$title" "$*" 2>&1)"; then # annotate the new task with the url, send the output back to the browser task +LATEST annotate "$QUTE_URL" echo "message-info '$msg'" >> "$QUTE_FIFO" diff --git a/misc/userscripts/view_in_mpv b/misc/userscripts/view_in_mpv index 9eb6ff7c6..f465fc4e4 100755 --- a/misc/userscripts/view_in_mpv +++ b/misc/userscripts/view_in_mpv @@ -50,7 +50,7 @@ msg() { MPV_COMMAND=${MPV_COMMAND:-mpv} # Warning: spaces in single flags are not supported MPV_FLAGS=${MPV_FLAGS:- --force-window --no-terminal --keep-open=yes --ytdl --ytdl-raw-options=yes-playlist=} -video_command=( "$MPV_COMMAND" $MPV_FLAGS ) +IFS=" " read -r -a video_command <<< "$MPV_COMMAND $MPV_FLAGS" js() { cat < Date: Sun, 3 Dec 2017 09:29:38 +0100 Subject: [PATCH 068/322] Use koalaman/shellcheck:latest --- scripts/dev/ci/travis_run.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index 70c41b0c3..f4960b808 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -16,14 +16,12 @@ elif [[ $TESTENV == eslint ]]; then eslint --color --report-unused-disable-directives . elif [[ $TESTENV == shellcheck ]]; then dev_scripts=$( find scripts/dev/ -name '*.sh' -print0 | xargs -0 ) - # false positive: we are using 'find -exec +' - # shellcheck disable=SC2038 userscripts=$( find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + | xargs ) IFS=" " read -r -a scripts <<< "$dev_scripts $userscripts" docker run \ -v "$PWD:/outside" \ -w /outside \ - koalaman/shellcheck:stable "${scripts[@]}" + koalaman/shellcheck:latest "${scripts[@]}" else args=() [[ $TRAVIS_OS_NAME == osx ]] && args=('--qute-bdd-webengine' '--no-xvfb') From 22e4a800a1c6080d29d41eeda229bc67b7bf314b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sun, 3 Dec 2017 10:50:54 +0100 Subject: [PATCH 069/322] Refactor format_json userscript to not parse 'ls' output The script now also works under MacOS --- misc/userscripts/format_json | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/misc/userscripts/format_json b/misc/userscripts/format_json index 10ff23e53..0d476b327 100755 --- a/misc/userscripts/format_json +++ b/misc/userscripts/format_json @@ -1,4 +1,5 @@ #!/bin/sh +set -euo pipefail # # Behavior: # Userscript for qutebrowser which will take the raw JSON text of the current @@ -19,28 +20,23 @@ # # Bryan Gilbert, 2017 +# do not run pygmentize on files larger than this amount of bytes +MAX_SIZE_PRETTIFY=10485760 # 10 MB # default style to monokai if none is provided STYLE=${1:-monokai} -# format json using jq -if ! FORMATTED_JSON="$(jq . "$QUTE_TEXT")" || [ -z "$FORMATTED_JSON" ]; then - echo "Invalid json, aborting..." - exit 1 + +TEMP_FILE="$(mktemp)" +jq . "$QUTE_TEXT" >"$TEMP_FILE" + +# try GNU stat first and then OSX stat if the former fails +FILE_SIZE=$( + stat --printf="%s" "$TEMP_FILE" 2>/dev/null || + stat -f%z "$TEMP_FILE" 2>/dev/null +) +if [ "$FILE_SIZE" -lt "$MAX_SIZE_PRETTIFY" ]; then + pygmentize -l json -f html -O full,style="$STYLE" <"$TEMP_FILE" >"${TEMP_FILE}_" + mv -f "${TEMP_FILE}_" "$TEMP_FILE" fi -# calculate the filesize of the json document -# parsing the output of ls should be fine in this case since we set the block size -# shellcheck disable=SC2012 -FILE_SIZE=$(ls -s --block-size=1048576 "$QUTE_TEXT" | cut -d' ' -f1) - -# use pygments to pretty-up the json (syntax highlight) if file is less than 10MB -if [ "$FILE_SIZE" -lt "10" ]; then - FORMATTED_JSON="$(echo "$FORMATTED_JSON" | pygmentize -l json -f html -O full,style="$STYLE")" -fi - -# create a temp file and write the formatted json to that file -TEMP_FILE="$(mktemp --suffix '.html')" -echo "$FORMATTED_JSON" > "$TEMP_FILE" - - # send the command to qutebrowser to open the new file containing the formatted json echo "open -t file://$TEMP_FILE" >> "$QUTE_FIFO" From 59c9a2b243204f4600018c12ccdad3fa40404071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Sun, 3 Dec 2017 11:30:59 +0100 Subject: [PATCH 070/322] Ignore shellcheck false positive --- scripts/dev/ci/travis_run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index f4960b808..74b8ae63f 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -16,6 +16,8 @@ elif [[ $TESTENV == eslint ]]; then eslint --color --report-unused-disable-directives . elif [[ $TESTENV == shellcheck ]]; then dev_scripts=$( find scripts/dev/ -name '*.sh' -print0 | xargs -0 ) + # false positive: we are using 'find -exec +' + # shellcheck disable=SC2038 userscripts=$( find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + | xargs ) IFS=" " read -r -a scripts <<< "$dev_scripts $userscripts" docker run \ From f07301cfb5f31e02b26e1865a8d71389f002c0d2 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 3 Dec 2017 12:48:29 +0100 Subject: [PATCH 071/322] Revert "Restart correctly after reporting crash." This reverts commit 7001f068b3ef29dce63feab19ee008d8c7e5ca27. --- qutebrowser/misc/crashdialog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/misc/crashdialog.py b/qutebrowser/misc/crashdialog.py index 3d50e5f29..9919386c4 100644 --- a/qutebrowser/misc/crashdialog.py +++ b/qutebrowser/misc/crashdialog.py @@ -348,7 +348,7 @@ class _CrashDialog(QDialog): "but you're currently running v{} - please " "update!".format(newest, qutebrowser.__version__)) text = '

'.join(lines) - self.finish() + self.hide() msgbox.information(self, "Report successfully sent!", text, on_finished=self.finish, plain_text=False) @@ -365,7 +365,7 @@ class _CrashDialog(QDialog): "qutebrowser.org " "by yourself.".format(msg)) text = '

'.join(lines) - self.finish() + self.hide() msgbox.information(self, "Report successfully sent!", text, on_finished=self.finish, plain_text=False) From 97054ca35de48294ff549b82d2d586bd7c7b28ce Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 3 Dec 2017 13:04:08 +0100 Subject: [PATCH 072/322] Don't hide report dialog early It looks like hiding it already causes it to be accepted. Fixes #1128 --- qutebrowser/misc/crashdialog.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qutebrowser/misc/crashdialog.py b/qutebrowser/misc/crashdialog.py index 9919386c4..54827211a 100644 --- a/qutebrowser/misc/crashdialog.py +++ b/qutebrowser/misc/crashdialog.py @@ -348,7 +348,6 @@ class _CrashDialog(QDialog): "but you're currently running v{} - please " "update!".format(newest, qutebrowser.__version__)) text = '

'.join(lines) - self.hide() msgbox.information(self, "Report successfully sent!", text, on_finished=self.finish, plain_text=False) @@ -365,7 +364,6 @@ class _CrashDialog(QDialog): "qutebrowser.org " "by yourself.".format(msg)) text = '

'.join(lines) - self.hide() msgbox.information(self, "Report successfully sent!", text, on_finished=self.finish, plain_text=False) From b610563e7fed16f6403925fb5d17eb2f9f9a70a7 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 3 Dec 2017 07:32:55 -0500 Subject: [PATCH 073/322] Don't show current window for :tab-give/:tab-take. Resolves #3144. --- qutebrowser/browser/commands.py | 2 +- qutebrowser/completion/completer.py | 8 +++-- qutebrowser/completion/models/miscmodels.py | 30 +++++++++++++++--- qutebrowser/mainwindow/mainwindow.py | 2 +- tests/unit/completion/test_completer.py | 2 +- tests/unit/completion/test_models.py | 35 ++++++++++++++++++--- 6 files changed, 65 insertions(+), 14 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 0de903004..bced4daf4 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -518,7 +518,7 @@ class CommandDispatcher: return newtab @cmdutils.register(instance='command-dispatcher', scope='window') - @cmdutils.argument('index', completion=miscmodels.buffer) + @cmdutils.argument('index', completion=miscmodels.other_buffer) def tab_take(self, index): """Take a tab from another window. diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index bc0e4991f..15c5d7f14 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -35,6 +35,7 @@ class CompletionInfo: config = attr.ib() keyconf = attr.ib() + win_id = attr.ib() class Completer(QObject): @@ -43,6 +44,7 @@ class Completer(QObject): Attributes: _cmd: The statusbar Command object this completer belongs to. + _win_id: The id of the window that owns this object. _timer: The timer used to trigger the completion update. _last_cursor_pos: The old cursor position so we avoid double completion updates. @@ -50,9 +52,10 @@ class Completer(QObject): _last_completion_func: The completion function used for the last text. """ - def __init__(self, cmd, parent=None): + def __init__(self, cmd, win_id, parent=None): super().__init__(parent) self._cmd = cmd + self._win_id = win_id self._timer = QTimer() self._timer.setSingleShot(True) self._timer.setInterval(0) @@ -250,7 +253,8 @@ class Completer(QObject): with debug.log_time(log.completion, 'Starting {} completion'.format(func.__name__)): info = CompletionInfo(config=config.instance, - keyconf=config.key_instance) + keyconf=config.key_instance, + win_id=self._win_id) model = func(*args, info=info) with debug.log_time(log.completion, 'Set completion model'): completion.set_model(model) diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 28ff0ddac..6175dcfb4 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -94,10 +94,10 @@ def session(*, info=None): # pylint: disable=unused-argument return model -def buffer(*, info=None): # pylint: disable=unused-argument - """A model to complete on open tabs across all windows. - - Used for switching the buffer command. +def _buffer(skip_win_id=None): + """Helper to get the completion model for buffer/other_buffer. + Args: + skip_win_id: The id of the window to skip, or None to include all. """ def delete_buffer(data): """Close the selected tab.""" @@ -109,6 +109,8 @@ def buffer(*, info=None): # pylint: disable=unused-argument model = completionmodel.CompletionModel(column_widths=(6, 40, 54)) for win_id in objreg.window_registry: + if skip_win_id and win_id == skip_win_id: + continue tabbed_browser = objreg.get('tabbed-browser', scope='window', window=win_id) if tabbed_browser.shutting_down: @@ -126,13 +128,31 @@ def buffer(*, info=None): # pylint: disable=unused-argument return model -def window(*, info=None): # pylint: disable=unused-argument +def buffer(*, info=None): # pylint: disable=unused-argument + """A model to complete on open tabs across all windows. + + Used for switching the buffer command. + """ + return _buffer() + + +def other_buffer(*, info): + """A model to complete on open tabs across all windows except the current. + + Used for the tab-take command. + """ + return _buffer(skip_win_id=info.win_id) + + +def window(*, info): """A model to complete on all open windows.""" model = completionmodel.CompletionModel(column_widths=(6, 30, 64)) windows = [] for win_id in objreg.window_registry: + if win_id == info.win_id: + continue tabbed_browser = objreg.get('tabbed-browser', scope='window', window=win_id) tab_titles = (tab.title() for tab in tabbed_browser.widgets()) diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index bf95d3e6a..5acec2384 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -320,7 +320,7 @@ class MainWindow(QWidget): def _init_completion(self): self._completion = completionwidget.CompletionView(self.win_id, self) cmd = objreg.get('status-command', scope='window', window=self.win_id) - completer_obj = completer.Completer(cmd, self._completion) + completer_obj = completer.Completer(cmd, self.win_id, self._completion) self._completion.selection_changed.connect( completer_obj.on_selection_changed) objreg.register('completion', self._completion, scope='window', diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index a32241621..0e8aabb93 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -65,7 +65,7 @@ def completer_obj(qtbot, status_command_stub, config_stub, monkeypatch, stubs, """Create the completer used for testing.""" monkeypatch.setattr(completer, 'QTimer', stubs.InstaTimer) config_stub.val.completion.show = 'auto' - return completer.Completer(status_command_stub, completion_widget_stub) + return completer.Completer(status_command_stub, 0, completion_widget_stub) @pytest.fixture(autouse=True) diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 9c767c102..8879f3201 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -191,7 +191,8 @@ def web_history_populated(web_history): @pytest.fixture def info(config_stub, key_config_stub): return completer.CompletionInfo(config=config_stub, - keyconf=key_config_stub) + keyconf=key_config_stub, + win_id=0) def test_command_completion(qtmodeltester, cmdutils_stub, configdata_stub, @@ -581,7 +582,33 @@ def test_tab_completion_delete(qtmodeltester, fake_web_tab, app_stub, QUrl('https://duckduckgo.com')] -def test_window_completion(qtmodeltester, fake_web_tab, tabbed_browser_stubs): +def test_other_buffer_completion(qtmodeltester, fake_web_tab, app_stub, + win_registry, tabbed_browser_stubs, info): + tabbed_browser_stubs[0].tabs = [ + fake_web_tab(QUrl('https://github.com'), 'GitHub', 0), + fake_web_tab(QUrl('https://wikipedia.org'), 'Wikipedia', 1), + fake_web_tab(QUrl('https://duckduckgo.com'), 'DuckDuckGo', 2), + ] + tabbed_browser_stubs[1].tabs = [ + fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0), + ] + info.win_id = 1 + model = miscmodels.other_buffer(info=info) + model.set_pattern('') + qtmodeltester.data_display_may_return_none = True + qtmodeltester.check(model) + + _check_completions(model, { + '0': [ + ('0/1', 'https://github.com', 'GitHub'), + ('0/2', 'https://wikipedia.org', 'Wikipedia'), + ('0/3', 'https://duckduckgo.com', 'DuckDuckGo') + ], + }) + + +def test_window_completion(qtmodeltester, fake_web_tab, tabbed_browser_stubs, + info): tabbed_browser_stubs[0].tabs = [ fake_web_tab(QUrl('https://github.com'), 'GitHub', 0), fake_web_tab(QUrl('https://wikipedia.org'), 'Wikipedia', 1), @@ -591,7 +618,8 @@ def test_window_completion(qtmodeltester, fake_web_tab, tabbed_browser_stubs): fake_web_tab(QUrl('https://wiki.archlinux.org'), 'ArchWiki', 0) ] - model = miscmodels.window() + info.win_id = 1 + model = miscmodels.window(info=info) model.set_pattern('') qtmodeltester.data_display_may_return_none = True qtmodeltester.check(model) @@ -600,7 +628,6 @@ def test_window_completion(qtmodeltester, fake_web_tab, tabbed_browser_stubs): 'Windows': [ ('0', 'window title - qutebrowser', 'GitHub, Wikipedia, DuckDuckGo'), - ('1', 'window title - qutebrowser', 'ArchWiki') ] }) From 38b2d42b4019d31febf298667041c6bbf6e6cd9e Mon Sep 17 00:00:00 2001 From: Ryan Farley Date: Sun, 3 Dec 2017 15:09:47 -0600 Subject: [PATCH 074/322] cleanup PYTEST_ADDOPTS for pytest subprocess See https://github.com/qutebrowser/qutebrowser/pull/3349 --- tests/unit/scripts/test_check_coverage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/scripts/test_check_coverage.py b/tests/unit/scripts/test_check_coverage.py index 6b18568c5..d7183111f 100644 --- a/tests/unit/scripts/test_check_coverage.py +++ b/tests/unit/scripts/test_check_coverage.py @@ -50,6 +50,7 @@ class CovtestHelper: def run(self): """Run pytest with coverage for the given module.py.""" coveragerc = str(self._testdir.tmpdir / 'coveragerc') + self._monkeypatch.delenv('PYTEST_ADDOPTS', raising=False) return self._testdir.runpytest('--cov=module', '--cov-config={}'.format(coveragerc), '--cov-report=xml', From a137a29ccea17fa0228e339182b656ed63090e27 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Sun, 3 Dec 2017 22:32:17 +0100 Subject: [PATCH 075/322] Style improvements This adds a blank line and makes Completer arguments keyword-only to make their meaning more clear. --- qutebrowser/completion/completer.py | 2 +- qutebrowser/completion/models/miscmodels.py | 1 + qutebrowser/mainwindow/mainwindow.py | 3 ++- tests/unit/completion/test_completer.py | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/qutebrowser/completion/completer.py b/qutebrowser/completion/completer.py index 15c5d7f14..edb1aca97 100644 --- a/qutebrowser/completion/completer.py +++ b/qutebrowser/completion/completer.py @@ -52,7 +52,7 @@ class Completer(QObject): _last_completion_func: The completion function used for the last text. """ - def __init__(self, cmd, win_id, parent=None): + def __init__(self, *, cmd, win_id, parent=None): super().__init__(parent) self._cmd = cmd self._win_id = win_id diff --git a/qutebrowser/completion/models/miscmodels.py b/qutebrowser/completion/models/miscmodels.py index 6175dcfb4..b1e599798 100644 --- a/qutebrowser/completion/models/miscmodels.py +++ b/qutebrowser/completion/models/miscmodels.py @@ -96,6 +96,7 @@ def session(*, info=None): # pylint: disable=unused-argument def _buffer(skip_win_id=None): """Helper to get the completion model for buffer/other_buffer. + Args: skip_win_id: The id of the window to skip, or None to include all. """ diff --git a/qutebrowser/mainwindow/mainwindow.py b/qutebrowser/mainwindow/mainwindow.py index 5acec2384..7c35f0529 100644 --- a/qutebrowser/mainwindow/mainwindow.py +++ b/qutebrowser/mainwindow/mainwindow.py @@ -320,7 +320,8 @@ class MainWindow(QWidget): def _init_completion(self): self._completion = completionwidget.CompletionView(self.win_id, self) cmd = objreg.get('status-command', scope='window', window=self.win_id) - completer_obj = completer.Completer(cmd, self.win_id, self._completion) + completer_obj = completer.Completer(cmd=cmd, win_id=self.win_id, + parent=self._completion) self._completion.selection_changed.connect( completer_obj.on_selection_changed) objreg.register('completion', self._completion, scope='window', diff --git a/tests/unit/completion/test_completer.py b/tests/unit/completion/test_completer.py index 0e8aabb93..012122937 100644 --- a/tests/unit/completion/test_completer.py +++ b/tests/unit/completion/test_completer.py @@ -65,7 +65,8 @@ def completer_obj(qtbot, status_command_stub, config_stub, monkeypatch, stubs, """Create the completer used for testing.""" monkeypatch.setattr(completer, 'QTimer', stubs.InstaTimer) config_stub.val.completion.show = 'auto' - return completer.Completer(status_command_stub, 0, completion_widget_stub) + return completer.Completer(cmd=status_command_stub, win_id=0, + parent=completion_widget_stub) @pytest.fixture(autouse=True) From 6b65d96fe101517592820f6e9b0efa64d575a294 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 06:32:54 +0100 Subject: [PATCH 076/322] Reformat comment --- scripts/link_pyqt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/link_pyqt.py b/scripts/link_pyqt.py index 82f8cbac5..ad7d45383 100644 --- a/scripts/link_pyqt.py +++ b/scripts/link_pyqt.py @@ -205,7 +205,7 @@ def main(): args = parser.parse_args() if args.tox: - #workaround for the lack of negative factors in tox.ini + # Workaround for the lack of negative factors in tox.ini if 'LINK_PYQT_SKIP' in os.environ: print('LINK_PYQT_SKIP set, exiting...') sys.exit(0) From 2c2d7fe7349cfef6272e3db9d1a9a52878bde88b Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 06:36:42 +0100 Subject: [PATCH 077/322] Copy-paste pylint commands for second environment Otherwise, tox 2.3.1 (shipped with various distributions) fails with: tox.ConfigError: ConfigError: substitution key 'posargs' not found --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2ada5ab58..a45e04c79 100644 --- a/tox.ini +++ b/tox.ini @@ -133,7 +133,8 @@ deps = -r{toxinidir}/misc/requirements/requirements-pylint.txt commands = {envpython} scripts/link_pyqt.py --tox {envdir} - {[testenv:pylint]commands} + {envpython} -m pylint scripts qutebrowser --output-format=colorized --reports=no {posargs} + {envpython} scripts/dev/run_pylint_on_tests.py {toxinidir} --output-format=colorized --reports=no {posargs} [testenv:pylint-master] basepython = python3 From 7ef64c0f87adecb51017327832e17ef64ea1049e Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 06:45:47 +0100 Subject: [PATCH 078/322] Read $PYTHON in every tox.ini environment See #2341 --- doc/install.asciidoc | 4 ++-- tox.ini | 39 ++++++++++----------------------------- 2 files changed, 12 insertions(+), 31 deletions(-) diff --git a/doc/install.asciidoc b/doc/install.asciidoc index 1dba5fa57..15718ec41 100644 --- a/doc/install.asciidoc +++ b/doc/install.asciidoc @@ -379,8 +379,8 @@ local Qt install instead of installing PyQt in the virtualenv. However, unless you have a new QtWebKit or QtWebEngine available, qutebrowser will not work. It also typically means you'll be using an older release of QtWebEngine. -On Windows, run `tox -e 'mkvenv-win' instead, however make sure that ONLY -Python3 is in your PATH before running tox. +On Windows, run `set PYTHON=C:\path\to\python.exe` (CMD) or ``$Env:PYTHON = +"..."` (Powershell) first. Creating a wrapper script ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tox.ini b/tox.ini index a45e04c79..e59e8b660 100644 --- a/tox.ini +++ b/tox.ini @@ -35,16 +35,7 @@ commands = # other envs [testenv:mkvenv] -basepython = python3 -commands = {envpython} scripts/link_pyqt.py --tox {envdir} -envdir = {toxinidir}/.venv -usedevelop = true -deps = - -r{toxinidir}/requirements.txt - -# This is used for Windows, since binary name is different -[testenv:mkvenv-win] -basepython = python.exe +basepython = {env:PYTHON:python3} commands = {envpython} scripts/link_pyqt.py --tox {envdir} envdir = {toxinidir}/.venv usedevelop = true @@ -61,7 +52,7 @@ deps = {[testenv:mkvenv]deps} # Virtualenv with PyQt5 from PyPI [testenv:mkvenv-pypi] -basepython = python3 +basepython = {env:PYTHON:python3} envdir = {toxinidir}/.venv commands = {envpython} -c "" usedevelop = true @@ -69,19 +60,9 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-pyqt.txt -# This is used for Windows, since binary name is different -[testenv:mkvenv-win-pypi] -basepython = python.exe -commands = {envpython} -c "" -envdir = {toxinidir}/.venv -usedevelop = true -deps = - -r{toxinidir}/requirements.txt - -r{toxinidir}/misc/requirements/requirements-pyqt.txt - [testenv:misc] ignore_errors = true -basepython = python3 +basepython = {env:PYTHON:python3} # For global .gitignore files passenv = HOME deps = @@ -91,7 +72,7 @@ commands = {envpython} scripts/dev/misc_checks.py spelling [testenv:vulture] -basepython = python3 +basepython = {env:PYTHON:python3} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-vulture.txt @@ -101,7 +82,7 @@ commands = {envpython} scripts/dev/run_vulture.py [testenv:vulture-pyqtlink] -basepython = python3 +basepython = {env:PYTHON:python3} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/misc/requirements/requirements-vulture.txt @@ -137,7 +118,7 @@ commands = {envpython} scripts/dev/run_pylint_on_tests.py {toxinidir} --output-format=colorized --reports=no {posargs} [testenv:pylint-master] -basepython = python3 +basepython = {env:PYTHON:python3} passenv = {[testenv:pylint]passenv} deps = -r{toxinidir}/requirements.txt @@ -149,7 +130,7 @@ commands = {envpython} scripts/dev/run_pylint_on_tests.py --output-format=colorized --reports=no {posargs} [testenv:flake8] -basepython = python3 +basepython = {env:PYTHON:python3} passenv = deps = -r{toxinidir}/requirements.txt @@ -158,7 +139,7 @@ commands = {envpython} -m flake8 {posargs:qutebrowser tests scripts} [testenv:pyroma] -basepython = python3 +basepython = {env:PYTHON:python3} passenv = deps = -r{toxinidir}/misc/requirements/requirements-pyroma.txt @@ -166,7 +147,7 @@ commands = {envdir}/bin/pyroma . [testenv:check-manifest] -basepython = python3 +basepython = {env:PYTHON:python3} passenv = deps = -r{toxinidir}/misc/requirements/requirements-check-manifest.txt @@ -174,7 +155,7 @@ commands = {envdir}/bin/check-manifest --ignore 'qutebrowser/git-commit-id,qutebrowser/html/doc,qutebrowser/html/doc/*,*/__pycache__' [testenv:docs] -basepython = python3 +basepython = {env:PYTHON:python3} whitelist_externals = git passenv = TRAVIS TRAVIS_PULL_REQUEST deps = From a3f57b9a9b32ec64acc1307e410fd5f274e998fb Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:14 +0100 Subject: [PATCH 079/322] Update flake8-builtins from 1.0 to 1.0.post0 --- misc/requirements/requirements-flake8.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-flake8.txt b/misc/requirements/requirements-flake8.txt index a031778ba..37ceb7f31 100644 --- a/misc/requirements/requirements-flake8.txt +++ b/misc/requirements/requirements-flake8.txt @@ -3,7 +3,7 @@ attrs==17.3.0 flake8==3.5.0 flake8-bugbear==17.4.0 -flake8-builtins==1.0 +flake8-builtins==1.0.post0 flake8-comprehensions==1.4.1 flake8-copyright==0.2.0 flake8-debugger==3.0.0 From 905748f2d0fef6dd503b0eba1bd3b3fa598a3fc6 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:16 +0100 Subject: [PATCH 080/322] Update setuptools from 38.2.1 to 38.2.3 --- misc/requirements/requirements-pip.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pip.txt b/misc/requirements/requirements-pip.txt index 4f4b0986c..8ca5a867e 100644 --- a/misc/requirements/requirements-pip.txt +++ b/misc/requirements/requirements-pip.txt @@ -3,6 +3,6 @@ appdirs==1.4.3 packaging==16.8 pyparsing==2.2.0 -setuptools==38.2.1 +setuptools==38.2.3 six==1.11.0 wheel==0.30.0 From a3612a624a651147af6ac9cb98dc45498589146e Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:17 +0100 Subject: [PATCH 081/322] Update altgraph from 0.14 to 0.15 --- misc/requirements/requirements-pyinstaller.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pyinstaller.txt b/misc/requirements/requirements-pyinstaller.txt index e542e4243..b2803311f 100644 --- a/misc/requirements/requirements-pyinstaller.txt +++ b/misc/requirements/requirements-pyinstaller.txt @@ -1,6 +1,6 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py -altgraph==0.14 +altgraph==0.15 future==0.16.0 macholib==1.8 pefile==2017.11.5 From 96b6f7c4434c3070e7d4ffc36138d37c502e37a0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:19 +0100 Subject: [PATCH 082/322] Update macholib from 1.8 to 1.9 --- misc/requirements/requirements-pyinstaller.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pyinstaller.txt b/misc/requirements/requirements-pyinstaller.txt index b2803311f..f65e8c62a 100644 --- a/misc/requirements/requirements-pyinstaller.txt +++ b/misc/requirements/requirements-pyinstaller.txt @@ -2,6 +2,6 @@ altgraph==0.15 future==0.16.0 -macholib==1.8 +macholib==1.9 pefile==2017.11.5 -e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller From f7ccb8061b7acb00b68a19d625129c34c3c0bee1 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:20 +0100 Subject: [PATCH 083/322] Update pyroma from 2.2 to 2.3 --- misc/requirements/requirements-pyroma.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pyroma.txt b/misc/requirements/requirements-pyroma.txt index d6ed0c190..241273169 100644 --- a/misc/requirements/requirements-pyroma.txt +++ b/misc/requirements/requirements-pyroma.txt @@ -1,4 +1,4 @@ # This file is automatically generated by scripts/dev/recompile_requirements.py docutils==0.14 -pyroma==2.2 +pyroma==2.3 From a0caa2b7b1161126b0f81e83d7350714ce1a3edd Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:22 +0100 Subject: [PATCH 084/322] Update hypothesis from 3.38.5 to 3.40.1 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 358dd72a6..a359d0662 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -11,7 +11,7 @@ fields==5.0.0 Flask==0.12.2 glob2==0.6 hunter==2.0.2 -hypothesis==3.38.5 +hypothesis==3.40.1 itsdangerous==0.24 # Jinja2==2.9.6 Mako==1.0.7 From 71095da975e37c91f44170630b213161bd076930 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:23 +0100 Subject: [PATCH 085/322] Update pytest from 3.2.5 to 3.3.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index a359d0662..cd00dd3e9 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -20,7 +20,7 @@ parse==1.8.2 parse-type==0.4.2 py==1.5.2 py-cpuinfo==3.3.0 -pytest==3.2.5 +pytest==3.3.0 pytest-bdd==2.19.0 pytest-benchmark==3.1.1 pytest-catchlog==1.2.2 From 956c257d196a251cb3f303908dfbebf3affda465 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 4 Dec 2017 16:09:25 +0100 Subject: [PATCH 086/322] Update pytest-travis-fold from 1.2.0 to 1.3.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index cd00dd3e9..37fd891db 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -31,7 +31,7 @@ pytest-mock==1.6.3 pytest-qt==2.3.0 pytest-repeat==0.4.1 pytest-rerunfailures==3.1 -pytest-travis-fold==1.2.0 +pytest-travis-fold==1.3.0 pytest-xvfb==1.0.0 PyVirtualDisplay==0.2.1 six==1.11.0 From 2cdc32ca5840b4f4804c97ef1ff6c85053f10714 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 16:55:57 +0100 Subject: [PATCH 087/322] Remove pytest-catchlog --- misc/requirements/requirements-tests-git.txt | 6 ------ misc/requirements/requirements-tests.txt | 1 - misc/requirements/requirements-tests.txt-raw | 1 - tests/helpers/logfail.py | 17 ++++----------- tests/helpers/test_logfail.py | 22 -------------------- tests/unit/utils/test_log.py | 6 +++--- 6 files changed, 7 insertions(+), 46 deletions(-) diff --git a/misc/requirements/requirements-tests-git.txt b/misc/requirements/requirements-tests-git.txt index 6b31140bb..6681dd15e 100644 --- a/misc/requirements/requirements-tests-git.txt +++ b/misc/requirements/requirements-tests-git.txt @@ -12,12 +12,6 @@ git+https://github.com/jenisys/parse_type.git hg+https://bitbucket.org/pytest-dev/py git+https://github.com/pytest-dev/pytest.git@features git+https://github.com/pytest-dev/pytest-bdd.git - -# This is broken at the moment because logfail tries to access -# LogCaptureHandler -# git+https://github.com/eisensheng/pytest-catchlog.git -pytest-catchlog==1.2.2 - git+https://github.com/pytest-dev/pytest-cov.git git+https://github.com/pytest-dev/pytest-faulthandler.git git+https://github.com/pytest-dev/pytest-instafail.git diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 37fd891db..58644a500 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -23,7 +23,6 @@ py-cpuinfo==3.3.0 pytest==3.3.0 pytest-bdd==2.19.0 pytest-benchmark==3.1.1 -pytest-catchlog==1.2.2 pytest-cov==2.5.1 pytest-faulthandler==1.3.1 pytest-instafail==0.3.0 diff --git a/misc/requirements/requirements-tests.txt-raw b/misc/requirements/requirements-tests.txt-raw index bc44bc8e1..121689980 100644 --- a/misc/requirements/requirements-tests.txt-raw +++ b/misc/requirements/requirements-tests.txt-raw @@ -7,7 +7,6 @@ hypothesis pytest pytest-bdd pytest-benchmark -pytest-catchlog pytest-cov pytest-faulthandler pytest-instafail diff --git a/tests/helpers/logfail.py b/tests/helpers/logfail.py index 3d8e3afb8..ba7ed24b8 100644 --- a/tests/helpers/logfail.py +++ b/tests/helpers/logfail.py @@ -22,16 +22,7 @@ import logging import pytest - -try: - import pytest_catchlog as catchlog_mod -except ImportError: - # When using pytest for pyflakes/pep8/..., the plugin won't be available - # but conftest.py will still be loaded. - # - # However, LogFailHandler.emit will never be used in that case, so we just - # ignore the ImportError. - pass +import _pytest.logging class LogFailHandler(logging.Handler): @@ -50,8 +41,8 @@ class LogFailHandler(logging.Handler): return for h in root_logger.handlers: - if isinstance(h, catchlog_mod.LogCaptureHandler): - catchlog_handler = h + if isinstance(h, _pytest.logging.LogCaptureHandler): + capture_handler = h break else: # The LogCaptureHandler is not available anymore during fixture @@ -59,7 +50,7 @@ class LogFailHandler(logging.Handler): return if (logger.level == record.levelno or - catchlog_handler.level == record.levelno): + capture_handler.level == record.levelno): # caplog.at_level(...) was used with the level of this message, # i.e. it was expected. return diff --git a/tests/helpers/test_logfail.py b/tests/helpers/test_logfail.py index b95dec1d6..48aaaa201 100644 --- a/tests/helpers/test_logfail.py +++ b/tests/helpers/test_logfail.py @@ -23,7 +23,6 @@ import logging import pytest -import pytest_catchlog def test_log_debug(): @@ -64,24 +63,3 @@ def test_log_expected_wrong_logger(caplog): with pytest.raises(pytest.fail.Exception): with caplog.at_level(logging.ERROR, logger): logging.error('foo') - - -@pytest.fixture -def skipping_fixture(): - pytest.skip("Skipping to test caplog workaround.") - - -def test_caplog_bug_workaround_1(caplog, skipping_fixture): - pass - - -def test_caplog_bug_workaround_2(): - """Make sure caplog_bug_workaround works correctly after a skipped test. - - There should be only one capturelog handler. - """ - caplog_handler = None - for h in logging.getLogger().handlers: - if isinstance(h, pytest_catchlog.LogCaptureHandler): - assert caplog_handler is None - caplog_handler = h diff --git a/tests/unit/utils/test_log.py b/tests/unit/utils/test_log.py index 30dd5d634..2ab51002d 100644 --- a/tests/unit/utils/test_log.py +++ b/tests/unit/utils/test_log.py @@ -27,7 +27,7 @@ import warnings import attr import pytest -import pytest_catchlog +import _pytest.logging from qutebrowser.utils import log from qutebrowser.misc import utilcmds @@ -66,11 +66,11 @@ def restore_loggers(): while root_logger.handlers: h = root_logger.handlers[0] root_logger.removeHandler(h) - if not isinstance(h, pytest_catchlog.LogCaptureHandler): + if not isinstance(h, _pytest.logging.LogCaptureHandler): h.close() root_logger.setLevel(original_logging_level) for h in root_handlers: - if not isinstance(h, pytest_catchlog.LogCaptureHandler): + if not isinstance(h, _pytest.logging.LogCaptureHandler): # https://github.com/qutebrowser/qutebrowser/issues/856 root_logger.addHandler(h) logging._acquireLock() From 6973a703c54783b774079bab40f4d75e64262e17 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 16:56:39 +0100 Subject: [PATCH 088/322] Add pluggy to requirements --- misc/requirements/requirements-tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 58644a500..119891725 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -18,6 +18,7 @@ Mako==1.0.7 # MarkupSafe==1.0 parse==1.8.2 parse-type==0.4.2 +pluggy==0.6.0 py==1.5.2 py-cpuinfo==3.3.0 pytest==3.3.0 From b6466b74108d5087f3b5659d75dd4ab0e0e1ac93 Mon Sep 17 00:00:00 2001 From: Josefson Fraga Date: Mon, 4 Dec 2017 13:08:56 -0300 Subject: [PATCH 089/322] revision 2 --- scripts/hist_importer.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 79c723c58..36f151e6f 100755 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -2,7 +2,7 @@ # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: # Copyright 2017 Florian Bruhin (The Compiler) -# Copyright 2014-2017 Josefson Souza +# Copyright 2017 Josefson Souza # This file is part of qutebrowser. # @@ -32,17 +32,17 @@ import os def parse(): """Parse command line arguments.""" description = ("This program is meant to extract browser history from your" - "previous browser and import them into qutebrowser.") - epilog = ("Databases:\n\tQutebrowser: Is named 'history.sqlite' and can be" + " previous browser and import them into qutebrowser.") + epilog = ("Databases:\n\n\tqutebrowser: Is named 'history.sqlite' and can be" " found at your --basedir. In order to find where your basedir" " is you can run ':open qute:version' inside qutebrowser." - "\n\tFirerox: Is named 'places.sqlite', and can be found at your" - "system\"s profile folder. Check this link for where it is locat" - "ed: http://kb.mozillazine.org/Profile_folder" - "\n\tChrome: Is named 'History', and can be found at the respec" - "tive User Data Directory. Check this link for where it is locat" - "ed: https://chromium.googlesource.com/chromium/src/+/master/" - "docs/user_data_dir.md\n\n" + "\n\n\tFirefox: Is named 'places.sqlite', and can be found at your" + " system's profile folder. Check this link for where it is " + "located: http://kb.mozillazine.org/Profile_folder" + "\n\n\tChrome: Is named 'History', and can be found at the " + "respective User Data Directory. Check this link for where it is" + "located: https://chromium.googlesource.com/chromium/src/+/" + "master/docs/user_data_dir.md\n\n" "Example: hist_importer.py -b firefox -s /Firefox/Profile/" "places.sqlite -d /qutebrowser/data/history.sqlite") parser = argparse.ArgumentParser( @@ -55,7 +55,7 @@ def parse(): type=str, help='Source: Full path to the sqlite data' 'base file from the source browser.') parser.add_argument('-d', '--dest', dest='dest', required=True, type=str, - help='Destination: The full path to the qutebrowser ' + help='\nDestination: Full path to the qutebrowser ' 'sqlite database') return parser.parse_args() @@ -66,10 +66,9 @@ def open_db(data_base): conn = sqlite3.connect(data_base) return conn else: - raise sys.exit('\nDataBaseNotFound: There was some error trying to to' + raise sys.exit('DataBaseNotFound: There was some error trying to to' ' connect with the [{}] database. Verify if the' - ' filepath is correct or is being used.' - .format(data_base)) + ' given file path is correct.'.format(data_base)) def extract(source, query): @@ -91,8 +90,7 @@ def extract(source, query): return history except sqlite3.OperationalError as op_e: print('\nCould not perform queries into the source database: {}' - '\nBrowser version is not supported as it have a different sql' - ' schema.'.format(op_e)) + .format(op_e)) def clean(history): From 4467f51e42275b1662b6aab7d7de073a5b0e6b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Mon, 4 Dec 2017 18:15:02 +0100 Subject: [PATCH 090/322] Use 'language: generic' for shellcheck, fix typo, correct indentation --- .travis.yml | 1 + misc/userscripts/rss | 4 ++-- scripts/dev/ci/travis_run.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37387b620..0ea8218a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,6 +53,7 @@ matrix: python: null node_js: "lts/*" - os: linux + language: generic env: TESTENV=shellcheck services: docker fast_finish: true diff --git a/misc/userscripts/rss b/misc/userscripts/rss index 6680a259e..f8feebee7 100755 --- a/misc/userscripts/rss +++ b/misc/userscripts/rss @@ -57,7 +57,7 @@ notice () { # Update a database of a feed and open new URLs read_items () { - cd read_urls || return + cd read_urls || return 1 feed_file="$(echo "$1" | tr -d /)" feed_temp_file="$(mktemp "$feed_file.tmp.XXXXXXXXXX")" feed_new_items="$(mktemp "$feed_file.new.XXXXXXXXXX")" @@ -85,7 +85,7 @@ if [ ! -d "$config_dir/read_urls" ]; then mkdir -p "$config_dir/read_urls" fi -cd "$config_dir" || exit +cd "$config_dir" || exit 1 if [ $# != 0 ]; then for arg in "$@"; do diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index 74b8ae63f..c0f59dc1a 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -22,7 +22,7 @@ elif [[ $TESTENV == shellcheck ]]; then IFS=" " read -r -a scripts <<< "$dev_scripts $userscripts" docker run \ -v "$PWD:/outside" \ - -w /outside \ + -w /outside \ koalaman/shellcheck:latest "${scripts[@]}" else args=() From 86c37538d71e2937386ae89089191018231c7998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Mon, 4 Dec 2017 18:29:55 +0100 Subject: [PATCH 091/322] Simply search for shell scripts to search Use 2 simpler find commands and redirect the output to a temporary file. --- scripts/dev/ci/travis_run.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/dev/ci/travis_run.sh b/scripts/dev/ci/travis_run.sh index c0f59dc1a..b7d44968e 100644 --- a/scripts/dev/ci/travis_run.sh +++ b/scripts/dev/ci/travis_run.sh @@ -15,11 +15,11 @@ elif [[ $TESTENV == eslint ]]; then cd qutebrowser/javascript || exit 1 eslint --color --report-unused-disable-directives . elif [[ $TESTENV == shellcheck ]]; then - dev_scripts=$( find scripts/dev/ -name '*.sh' -print0 | xargs -0 ) - # false positive: we are using 'find -exec +' - # shellcheck disable=SC2038 - userscripts=$( find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + | xargs ) - IFS=" " read -r -a scripts <<< "$dev_scripts $userscripts" + SCRIPTS=$( mktemp ) + find scripts/dev/ -name '*.sh' >"$SCRIPTS" + find misc/userscripts/ -type f -exec grep -lE '[/ ][bd]ash$|[/ ]sh$|[/ ]ksh$' {} + >>"$SCRIPTS" + mapfile -t scripts <"$SCRIPTS" + rm -f "$SCRIPTS" docker run \ -v "$PWD:/outside" \ -w /outside \ From 3a04de62ae4a5dde8aabd4edcc7a7e584c113aae Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 19:01:21 +0100 Subject: [PATCH 092/322] Recompile requirements --- misc/requirements/requirements-tests.txt | 2 +- misc/requirements/requirements-tox.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 119891725..46905f497 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -13,7 +13,7 @@ glob2==0.6 hunter==2.0.2 hypothesis==3.40.1 itsdangerous==0.24 -# Jinja2==2.9.6 +# Jinja2==2.10 Mako==1.0.7 # MarkupSafe==1.0 parse==1.8.2 diff --git a/misc/requirements/requirements-tox.txt b/misc/requirements/requirements-tox.txt index 1308c8afd..d2b3a719b 100644 --- a/misc/requirements/requirements-tox.txt +++ b/misc/requirements/requirements-tox.txt @@ -2,5 +2,6 @@ pluggy==0.6.0 py==1.5.2 +six==1.11.0 tox==2.9.1 virtualenv==15.1.0 From 28caddf3c1700f77c78173b3677abd71c8d9582a Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Mon, 4 Dec 2017 19:02:09 +0100 Subject: [PATCH 093/322] delay added, text changed --- qutebrowser/html/back.html | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index b6190feda..cfaae589b 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -21,6 +21,15 @@ function go_forward() { history.forward(); } +function prepare_restore() { + if (!document.hidden) { + go_back(); + break; + } + + document.addEventListener("visibilitychange", go_back); +} + // there are three states // default: register focus listener, // on focus: go back and switch to the state forward @@ -38,17 +47,14 @@ switch (history.state) { go_forward(); break; default: - if (!document.hidden) { - go_back(); - break; - } - - document.addEventListener("visibilitychange", go_back); + setTimeout(prepare_restore, 1000); break; } {% endblock %} {% block content %} -

If you see this site something went wrong, or you reached the end of the history, or you disabled javascript.

+

Loading suspended page...
+
+If nothing happens, something went wrong or you disabled JavaScript.

{% endblock %} From 02104a318e9c8dfea690447a544f84c2dd19197b Mon Sep 17 00:00:00 2001 From: "mhm@mhm.com" Date: Mon, 4 Dec 2017 19:03:12 +0100 Subject: [PATCH 094/322] delay added, text changed --- qutebrowser/html/back.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index cfaae589b..46945ab65 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -24,7 +24,7 @@ function go_forward() { function prepare_restore() { if (!document.hidden) { go_back(); - break; + return; } document.addEventListener("visibilitychange", go_back); From 9675ea93eebb51bcb80480a248ad209fc957dc29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Chavant?= Date: Mon, 4 Dec 2017 20:31:28 +0100 Subject: [PATCH 095/322] Do not call pip in travis_install.sh when TESTENV=shellcheck --- scripts/dev/ci/travis_install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh index 18ac44048..04b118b9b 100644 --- a/scripts/dev/ci/travis_install.sh +++ b/scripts/dev/ci/travis_install.sh @@ -96,6 +96,8 @@ case $TESTENV in eslint) npm_install eslint ;; + shellcheck) + ;; *) pip_install pip pip_install -r misc/requirements/requirements-tox.txt From b554e1f7639d340fa704396b61cd7346914a28f6 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 22:07:23 +0100 Subject: [PATCH 096/322] tests: Add after= argument to wait_for --- tests/end2end/fixtures/testprocess.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/end2end/fixtures/testprocess.py b/tests/end2end/fixtures/testprocess.py index a4b136193..c210f6fe7 100644 --- a/tests/end2end/fixtures/testprocess.py +++ b/tests/end2end/fixtures/testprocess.py @@ -333,7 +333,7 @@ class Process(QObject): else: return value == expected - def _wait_for_existing(self, override_waited_for, **kwargs): + def _wait_for_existing(self, override_waited_for, after, **kwargs): """Check if there are any line in the history for wait_for. Return: either the found line or None. @@ -345,7 +345,15 @@ class Process(QObject): value = getattr(line, key) matches.append(self._match_data(value, expected)) - if all(matches) and (not line.waited_for or override_waited_for): + if after is None: + too_early = False + else: + too_early = ((line.timestamp, line.msecs) < + (after.timestamp, after.msecs)) + + if (all(matches) and + (not line.waited_for or override_waited_for) and + not too_early): # If we waited for this line, chances are we don't mean the # same thing the next time we use wait_for and it matches # this line again. @@ -422,7 +430,7 @@ class Process(QObject): pass def wait_for(self, timeout=None, *, override_waited_for=False, - do_skip=False, divisor=1, **kwargs): + do_skip=False, divisor=1, after=None, **kwargs): """Wait until a given value is found in the data. Keyword arguments to this function get interpreted as attributes of the @@ -435,6 +443,7 @@ class Process(QObject): again. do_skip: If set, call pytest.skip on a timeout. divisor: A factor to decrease the timeout by. + after: If it's an existing line, ensure it's after the given one. Return: The matched line. @@ -456,7 +465,8 @@ class Process(QObject): for key in kwargs: assert key in self.KEYS - existing = self._wait_for_existing(override_waited_for, **kwargs) + existing = self._wait_for_existing(override_waited_for, after, + **kwargs) if existing is not None: return existing else: From a8f4444c24f785dfc24c107f67d5f8a2208d95d1 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 4 Dec 2017 22:07:49 +0100 Subject: [PATCH 097/322] tests: Show more of the message --- tests/end2end/fixtures/testprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/fixtures/testprocess.py b/tests/end2end/fixtures/testprocess.py index c210f6fe7..bc987043f 100644 --- a/tests/end2end/fixtures/testprocess.py +++ b/tests/end2end/fixtures/testprocess.py @@ -371,7 +371,7 @@ class Process(QObject): __tracebackhide__ = lambda e: e.errisinstance(WaitForTimeout) message = kwargs.get('message', None) if message is not None: - elided = quteutils.elide(repr(message), 50) + elided = quteutils.elide(repr(message), 100) self._log("\n----> Waiting for {} in the log".format(elided)) spy = QSignalSpy(self.new_data) From 62228752aae48367b9393288f0b7cf90cace1401 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 5 Dec 2017 07:05:07 +0100 Subject: [PATCH 098/322] Fix most end2end tests with Qt 5.10 For some reason, if we don't wait for about:blank to be fully loaded with Qt 5.10, we get the next LoadStatus.finished notification with about:blank as URL. This is most likely caused by the changes in https://codereview.qt-project.org/#/c/202924/ See #3003 --- tests/end2end/fixtures/quteprocess.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/end2end/fixtures/quteprocess.py b/tests/end2end/fixtures/quteprocess.py index 27c347ca4..454ae80a5 100644 --- a/tests/end2end/fixtures/quteprocess.py +++ b/tests/end2end/fixtures/quteprocess.py @@ -36,7 +36,7 @@ import pytest from PyQt5.QtCore import pyqtSignal, QUrl, qVersion from qutebrowser.misc import ipc -from qutebrowser.utils import log, utils, javascript +from qutebrowser.utils import log, utils, javascript, qtutils from helpers import utils as testutils from end2end.fixtures import testprocess @@ -687,9 +687,12 @@ class QuteProc(testprocess.Process): raise ValueError("Invalid URL {}: {}".format(url, qurl.errorString())) - if qurl == QUrl('about:blank'): + if (qurl == QUrl('about:blank') and + not qtutils.version_check('5.10', compiled=False)): # For some reason, we don't get a LoadStatus.success for # about:blank sometimes. + # However, if we do this for Qt 5.10, we get general testsuite + # instability as site loads get reported with about:blank... pattern = "Changing title for idx * to 'about:blank'" else: # We really need the same representation that the webview uses in From 0ce9a355aed3a7df1932c0b7bda34aeb1e5d3ef4 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 5 Dec 2017 07:07:13 +0100 Subject: [PATCH 099/322] Fix download test with Qt 5.10 Not sure why this is needed (no prompt is shown otherwise), but it works like this. This probably is related to https://bugreports.qt.io/browse/QTBUG-63388 See #3003 --- tests/end2end/features/downloads.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/end2end/features/downloads.feature b/tests/end2end/features/downloads.feature index 69f47603b..833f63cb4 100644 --- a/tests/end2end/features/downloads.feature +++ b/tests/end2end/features/downloads.feature @@ -536,7 +536,7 @@ Feature: Downloading things from a website. And I open data/downloads/download.bin without waiting And I wait for the download prompt for "*" And I run :prompt-accept (tmpdir)(dirsep)downloads - And I open data/downloads/download.bin without waiting + And I open data/downloads/download2.bin without waiting And I wait for the download prompt for "*" And I directly open the download And I open data/downloads/download.bin without waiting From 29c2e7b45f75cf6b1b3cdd4fb6cee341f20c7601 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 5 Dec 2017 07:09:43 +0100 Subject: [PATCH 100/322] Skip :follow-selected tests on Qt 5.10 See #3003, #2635 --- tests/end2end/features/search.feature | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/end2end/features/search.feature b/tests/end2end/features/search.feature index 3778f963d..56fcca207 100644 --- a/tests/end2end/features/search.feature +++ b/tests/end2end/features/search.feature @@ -225,11 +225,15 @@ Feature: Searching on a page Then the following tabs should be open: - data/search.html (active) + # Following a link selected via JS doesn't work in Qt 5.10 anymore. + @qt!=5.10 Scenario: Follow a manually selected link When I run :jseval --file (testdata)/search_select.js And I run :follow-selected Then data/hello.txt should be loaded + # Following a link selected via JS doesn't work in Qt 5.10 anymore. + @qt!=5.10 Scenario: Follow a manually selected link in a new tab When I run :window-only And I run :jseval --file (testdata)/search_select.js From 8a3437c6a4edabf2212a8e37e9e8490be18fa1b0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Tue, 5 Dec 2017 09:36:14 +0100 Subject: [PATCH 101/322] Adjust Debian install instructions --- doc/install.asciidoc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/install.asciidoc b/doc/install.asciidoc index 15718ec41..312e53176 100644 --- a/doc/install.asciidoc +++ b/doc/install.asciidoc @@ -35,7 +35,7 @@ Ubuntu 16.04 doesn't come with an up-to-date engine (a new enough QtWebKit, or QtWebEngine). However, it comes with Python 3.5, so you can <>. -Debian Stretch / Ubuntu 17.04 and newer +Debian Stretch / Ubuntu 17.04 and 17.10 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Those versions come with QtWebEngine in the repositories. This makes it possible @@ -54,7 +54,18 @@ Install the packages: # apt install ./qutebrowser_*_all.deb ---- -Some additional hints: +Debian Testing / Ubuntu 18.04 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Debian Testing, qutebrowser is in the official repositories, and you can +install it with apt: + +---- +# apt install qutebrowser +---- + +Additional hints +~~~~~~~~~~~~~~~~ - Alternatively, you can <> to get a newer QtWebEngine version. From 636f9edff62fb121bb52a1c25ec5903d24a18074 Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Tue, 5 Dec 2017 07:31:55 -0500 Subject: [PATCH 102/322] History completion by both URL and title. Resolves #1649. --- qutebrowser/completion/models/histcategory.py | 2 +- tests/unit/completion/test_histcategory.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index b993b40de..fe89dc79b 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -47,7 +47,7 @@ class HistoryCategory(QSqlQueryModel): "FROM CompletionHistory", # the incoming pattern will have literal % and _ escaped with '\' # we need to tell sql to treat '\' as an escape character - "WHERE (url LIKE :pat escape '\\' or title LIKE :pat escape '\\')", + "WHERE ((url || title) LIKE :pat escape '\\')", self._atime_expr(), "ORDER BY last_atime DESC", ]), forward_only=False) diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index 834b3a5a3..b87eb6ac2 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -78,6 +78,10 @@ def hist(init_sql, config_stub): ("can't", [("can't touch this", ''), ('a', '')], [("can't touch this", '')]), + + ("ample itle", + [('example.com', 'title'), ('example.com', 'nope')], + [('example.com', 'title')]), ]) def test_set_pattern(pattern, before, after, model_validator, hist): """Validate the filtering and sorting results of set_pattern.""" From 00a09354c3231d78700187a33ac49d2bbf1aca30 Mon Sep 17 00:00:00 2001 From: Justin Partain Date: Tue, 5 Dec 2017 08:28:10 -0500 Subject: [PATCH 103/322] Track number of active searches in tab, ignore all but most recent search callbacks --- qutebrowser/browser/webengine/webenginetab.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 7339cd422..b40d8b871 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -126,13 +126,19 @@ class WebEngineSearch(browsertab.AbstractSearch): def __init__(self, parent=None): super().__init__(parent) self._flags = QWebEnginePage.FindFlags(0) + self.num_of_searches = 0 def _find(self, text, flags, callback, caller): """Call findText on the widget.""" self.search_displayed = True + self.num_of_searches += 1 def wrapped_callback(found): """Wrap the callback to do debug logging.""" + self.num_of_searches -= 1 + if self.num_of_searches > 0: + return + found_text = 'found' if found else "didn't find" if flags: flag_text = 'with flags {}'.format(debug.qflags_key( From 7f81f0c0ab79f9082d807f23561060f12cd9d05f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 06:51:15 +0100 Subject: [PATCH 104/322] Always open session tabs in foreground This helps with issues with lazy sessions as document.hidden was set incorrectly. See #3345, #3366 --- qutebrowser/misc/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/misc/sessions.py b/qutebrowser/misc/sessions.py index 237e46189..bfd73ef32 100644 --- a/qutebrowser/misc/sessions.py +++ b/qutebrowser/misc/sessions.py @@ -422,7 +422,7 @@ class SessionManager(QObject): window=window.win_id) tab_to_focus = None for i, tab in enumerate(win['tabs']): - new_tab = tabbed_browser.tabopen() + new_tab = tabbed_browser.tabopen(background=False) self._load_tab(new_tab, tab) if tab.get('active', False): tab_to_focus = i From 7a6d568c8cd980e3b57c678070dfdea387245089 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 06:53:27 +0100 Subject: [PATCH 105/322] Remove blank line --- qutebrowser/html/back.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/html/back.html b/qutebrowser/html/back.html index 46945ab65..894427800 100644 --- a/qutebrowser/html/back.html +++ b/qutebrowser/html/back.html @@ -26,7 +26,7 @@ function prepare_restore() { go_back(); return; } - + document.addEventListener("visibilitychange", go_back); } From 58212a7b15864b2a1b3b6afa73972908032f0e02 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 06:56:12 +0100 Subject: [PATCH 106/322] Update docs --- doc/changelog.asciidoc | 2 ++ doc/help/settings.asciidoc | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 34ee4ff7c..19714700a 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -51,6 +51,8 @@ Added - New `:edit-command` command to edit the commandline in an editor. - New `tabs.persist_mode_on_change` setting to keep the current mode when switching tabs. +- New `session.lazy_restore` setting which allows to not load pages immediately + when restoring a session. Changed ~~~~~~~ diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 2da7a5008..967f6c4c6 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -223,6 +223,7 @@ |<>|Show a scrollbar. |<>|Enable smooth scrolling for web pages. |<>|Name of the session to save by default. +|<>|Load a restored tab as soon as it takes focus. |<>|Languages to use for spell checking. |<>|Hide the statusbar unless a message is shown. |<>|Padding (in pixels) for the statusbar. @@ -2565,6 +2566,14 @@ Type: <> Default: empty +[[session.lazy_restore]] +=== session.lazy_restore +Load a restored tab as soon as it takes focus. + +Type: <> + +Default: +pass:[false]+ + [[spellcheck.languages]] === spellcheck.languages Languages to use for spell checking. From 9c042e4313a7f1229e2bb8169f6151681937e8f9 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 06:58:22 +0100 Subject: [PATCH 107/322] Update changelog --- doc/changelog.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 19714700a..f734d96bc 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -91,6 +91,8 @@ Changed data dir, e.g. `~/.local/share/qutebrowser/js`. - The current/default bindings are now shown in the :bind completion. - Empty categories are now hidden in the `:open` completion. +- Search terms for URLs and titles can now be mixed when filtering the + completion. Fixed ~~~~~ From b326f1242766212fb593ec7c0adc4dc0e77ddc98 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 06:59:08 +0100 Subject: [PATCH 108/322] Mark editor test as flaky See #3367 --- tests/end2end/features/editor.feature | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 21f3df425..3e1be47bc 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -117,6 +117,8 @@ Feature: Opening external editors # Could not get signals working on Windows @posix + # There's no guarantee that the tab gets deleted... + @flaky Scenario: Spawning an editor and closing the tab When I set up a fake editor that waits And I open data/editor.html From 549a3a8f70686b1eb2e3a583c477482c94add605 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 07:41:41 +0100 Subject: [PATCH 109/322] Improve hist_importer messages --- scripts/hist_importer.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index 36f151e6f..ba3369f5c 100755 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -66,9 +66,7 @@ def open_db(data_base): conn = sqlite3.connect(data_base) return conn else: - raise sys.exit('DataBaseNotFound: There was some error trying to to' - ' connect with the [{}] database. Verify if the' - ' given file path is correct.'.format(data_base)) + sys.exit('The file {} does not exist.'.format(data_base)) def extract(source, query): @@ -89,8 +87,8 @@ def extract(source, query): conn.close() return history except sqlite3.OperationalError as op_e: - print('\nCould not perform queries into the source database: {}' - .format(op_e)) + sys.exit('Could not perform queries on the source database: ' + '{}'.format(op_e)) def clean(history): From 16e09d18fa1207ccb78d6274965388c91a901f4c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 07:42:07 +0100 Subject: [PATCH 110/322] Update changelog --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index f734d96bc..4acd841b2 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -53,6 +53,7 @@ Added switching tabs. - New `session.lazy_restore` setting which allows to not load pages immediately when restoring a session. +- New `hist_importer.py` script to import history from Firefox/Chromium. Changed ~~~~~~~ From a3ba7b9f60393445cce7e6128e9a82bcfade5f42 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 07:45:52 +0100 Subject: [PATCH 111/322] Reformat hist_importer epilog. --- scripts/hist_importer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/hist_importer.py b/scripts/hist_importer.py index ba3369f5c..f4ad47062 100755 --- a/scripts/hist_importer.py +++ b/scripts/hist_importer.py @@ -33,11 +33,11 @@ def parse(): """Parse command line arguments.""" description = ("This program is meant to extract browser history from your" " previous browser and import them into qutebrowser.") - epilog = ("Databases:\n\n\tqutebrowser: Is named 'history.sqlite' and can be" - " found at your --basedir. In order to find where your basedir" - " is you can run ':open qute:version' inside qutebrowser." - "\n\n\tFirefox: Is named 'places.sqlite', and can be found at your" - " system's profile folder. Check this link for where it is " + epilog = ("Databases:\n\n\tqutebrowser: Is named 'history.sqlite' and can " + "be found at your --basedir. In order to find where your " + "basedir is you can run ':open qute:version' inside qutebrowser." + "\n\n\tFirefox: Is named 'places.sqlite', and can be found at " + "your system's profile folder. Check this link for where it is " "located: http://kb.mozillazine.org/Profile_folder" "\n\n\tChrome: Is named 'History', and can be found at the " "respective User Data Directory. Check this link for where it is" From 1a3f8662e6288cf46d0a9daf61d220718b94d448 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 07:56:59 +0100 Subject: [PATCH 112/322] Improve handling of cancelled search callbacks --- qutebrowser/browser/webengine/webenginetab.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py index 32bb95ca1..5a656e2b5 100644 --- a/qutebrowser/browser/webengine/webenginetab.py +++ b/qutebrowser/browser/webengine/webenginetab.py @@ -121,22 +121,33 @@ class WebEnginePrinting(browsertab.AbstractPrinting): class WebEngineSearch(browsertab.AbstractSearch): - """QtWebEngine implementations related to searching on the page.""" + """QtWebEngine implementations related to searching on the page. + + Attributes: + _flags: The QWebEnginePage.FindFlags of the last search. + _pending_searches: How many searches have been started but not called + back yet. + """ def __init__(self, parent=None): super().__init__(parent) self._flags = QWebEnginePage.FindFlags(0) - self.num_of_searches = 0 + self._pending_searches = 0 def _find(self, text, flags, callback, caller): """Call findText on the widget.""" self.search_displayed = True - self.num_of_searches += 1 + self._pending_searches += 1 def wrapped_callback(found): """Wrap the callback to do debug logging.""" - self.num_of_searches -= 1 - if self.num_of_searches > 0: + self._pending_searches -= 1 + if self._pending_searches > 0: + # See https://github.com/qutebrowser/qutebrowser/issues/2442 + # and https://github.com/qt/qtwebengine/blob/5.10/src/core/web_contents_adapter.cpp#L924-L934 + log.webview.debug("Ignoring cancelled search callback with " + "{} pending searches".format( + self._pending_searches)) return found_text = 'found' if found else "didn't find" From 7d7c8412500ae4e486d8d88c6db97e457d8921ce Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 08:01:39 +0100 Subject: [PATCH 113/322] Update changelog --- doc/changelog.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 4acd841b2..2302a65c4 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -112,6 +112,8 @@ Fixed in a URL. - Using e.g. `-s backend webkit` to set the backend now works correctly. - Fixed crash when closing the tab an external editor was opened in. +- When using `:search-next` before a search is finished, no warning about no + results being found is shown anymore. Deprecated ~~~~~~~~~~ From 129f97873a57d4b56b9c399e9da5c9da70dd880c Mon Sep 17 00:00:00 2001 From: Jimmy Date: Mon, 27 Nov 2017 20:07:16 +1300 Subject: [PATCH 114/322] Greasemonkey: add assert to tests scripts_for assumptions. And crash the users browsing session as a result of any accidental and totally, otherwise, non-fatal unforseen errors. --- qutebrowser/browser/webkit/webpage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 89b293869..02aa270d7 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -323,6 +323,8 @@ class BrowserPage(QWebPage): # also indicate a bug. log.greasemonkey.debug("Not running scripts for frame with no " "url: {}".format(frame)) + assert not toload, toload + for script in toload: if frame is self.mainFrame() or script.runs_on_sub_frames: log.webview.debug('Running GM script: {}'.format(script.name)) From ead108eeebadfc6e73b5927901eb155eebad028b Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 6 Dec 2017 20:27:56 +1300 Subject: [PATCH 115/322] fixup! Greasemonkey: Add run-at document-idle. --- qutebrowser/browser/greasemonkey.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 845b83d85..3fd01137f 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -193,8 +193,8 @@ class GreasemonkeyManager(QObject): match = functools.partial(fnmatch.fnmatch, url.toString(QUrl.FullyEncoded)) tester = (lambda script: - any([match(pat) for pat in script.includes]) and - not any([match(pat) for pat in script.excludes])) + any(match(pat) for pat in script.includes) and + not any(match(pat) for pat in script.excludes)) return MatchingScripts( url, [script for script in self._run_start if tester(script)], From 6b3e16b1630acfc8a78f4a8994d9ad2279450386 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Wed, 6 Dec 2017 20:34:29 +1300 Subject: [PATCH 116/322] Greasemonkey: mark failing no(sub)frames test as flaky. This test is supposed to ensure that user scripts don't run on iframes when the @noframes directive is set in the greasemonkey metadata. It is failing sometimes on travis but passing on local test runs. Personally I haven't actually ran the whole test suite through, just the javascript tests. It maybe be some stale state that only shows up when you run the whole suite. It may be some timing issue that only shows up on travis because ???. Hopefully this stops the red x from showing up on the PR. --- tests/end2end/features/javascript.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature index aaad84be3..944d2606d 100644 --- a/tests/end2end/features/javascript.feature +++ b/tests/end2end/features/javascript.feature @@ -139,6 +139,7 @@ Feature: Javascript stuff And I open data/hints/iframe.html Then the javascript message "Script is running on /data/hints/html/wrapped.html" should be logged + @flaky Scenario: Have a greasemonkey script running on noframes When I have a greasemonkey file saved for document-end with noframes set And I run :greasemonkey-reload From 0c792d228e4055c7195ee7b314bb522c8294dd84 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 11:12:25 +0100 Subject: [PATCH 117/322] Update docs --- doc/changelog.asciidoc | 2 ++ doc/help/commands.asciidoc | 7 +++++++ qutebrowser/browser/greasemonkey.py | 6 +++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 2302a65c4..16075db27 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -21,6 +21,8 @@ v1.1.0 (unreleased) Added ~~~~~ +- Initial support for Greasemonkey scripts. There are still some rough edges, + but many scripts should already work. - There's now a `misc/Makefile` file in releases, which should help distributions which package qutebrowser, as they can run something like `make -f misc/Makefile DESTDIR="$pkgdir" install` now. diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 0fb99f260..5d026bfca 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -54,6 +54,7 @@ It is possible to run or bind multiple commands by separating them with `;;`. |<>|Follow the selected text. |<>|Go forward in the history of the current tab. |<>|Toggle fullscreen mode. +|<>|Re-read Greasemonkey scripts from disk. |<>|Show help about a command or setting. |<>|Start hinting. |<>|Show browsing history. @@ -491,6 +492,12 @@ Toggle fullscreen mode. ==== optional arguments * +*-l*+, +*--leave*+: Only leave fullscreen if it was entered by the page. +[[greasemonkey-reload]] +=== greasemonkey-reload +Re-read Greasemonkey scripts from disk. + +The scripts are read from a 'greasemonkey' subdirectory in qutebrowser's data directory (see `:version`). + [[help]] === help Syntax: +:help [*--tab*] [*--bg*] [*--window*] ['topic']+ diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 3fd01137f..1a1ec6a24 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -149,7 +149,11 @@ class GreasemonkeyManager(QObject): @cmdutils.register(name='greasemonkey-reload', instance='greasemonkey') def load_scripts(self): - """Re-Read greasemonkey scripts from disk.""" + """Re-read Greasemonkey scripts from disk. + + The scripts are read from a 'greasemonkey' subdirectory in qutebrowser's + data directory (see `:version`). + """ self._run_start = [] self._run_end = [] self._run_idle = [] From dd63508be772679b4a0e684f8c00c1a65e3e9861 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 11:30:01 +0100 Subject: [PATCH 118/322] Add a greasemonkey.init() This also creates the greasemonkey directory if it doesn't exist yet, for discoverability. --- qutebrowser/app.py | 3 +-- qutebrowser/browser/greasemonkey.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/qutebrowser/app.py b/qutebrowser/app.py index c32a208ac..7029f8df5 100644 --- a/qutebrowser/app.py +++ b/qutebrowser/app.py @@ -492,8 +492,7 @@ def _init_modules(args, crash_handler): objreg.register('cache', diskcache) log.init.debug("Initializing Greasemonkey...") - gm_manager = greasemonkey.GreasemonkeyManager() - objreg.register('greasemonkey', gm_manager) + greasemonkey.init() log.init.debug("Misc initialization...") macros.init() diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 1a1ec6a24..669b9d0bc 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -29,7 +29,7 @@ import glob import attr from PyQt5.QtCore import pyqtSignal, QObject, QUrl -from qutebrowser.utils import log, standarddir, jinja +from qutebrowser.utils import log, standarddir, jinja, objreg from qutebrowser.commands import cmdutils @@ -209,3 +209,14 @@ class GreasemonkeyManager(QObject): def all_scripts(self): """Return all scripts found in the configured script directory.""" return self._run_start + self._run_end + self._run_idle + + +def init(): + """Initialize Greasemonkey support.""" + gm_manager = GreasemonkeyManager() + objreg.register('greasemonkey', gm_manager) + + try: + os.mkdir(_scripts_dir()) + except FileExistsError: + pass From 2633dcc0d572da84331362857872a3cd5af27e35 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 11:50:59 +0100 Subject: [PATCH 119/322] Fix lint --- qutebrowser/browser/greasemonkey.py | 24 ++++++------ .../browser/webengine/webenginesettings.py | 13 +++---- qutebrowser/browser/webengine/webview.py | 2 +- qutebrowser/browser/webkit/webpage.py | 4 +- .../javascript/greasemonkey_wrapper.js | 38 +++++++++---------- tests/end2end/features/javascript.feature | 12 +++--- tests/end2end/features/test_javascript_bdd.py | 4 +- tests/unit/javascript/test_greasemonkey.py | 6 +-- 8 files changed, 51 insertions(+), 52 deletions(-) diff --git a/qutebrowser/browser/greasemonkey.py b/qutebrowser/browser/greasemonkey.py index 669b9d0bc..9a82d6a93 100644 --- a/qutebrowser/browser/greasemonkey.py +++ b/qutebrowser/browser/greasemonkey.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""Load, parse and make avalaible greasemonkey scripts.""" +"""Load, parse and make available Greasemonkey scripts.""" import re import os @@ -39,6 +39,7 @@ def _scripts_dir(): class GreasemonkeyScript: + """Container class for userscripts, parses metadata blocks.""" def __init__(self, properties, code): @@ -72,15 +73,15 @@ class GreasemonkeyScript: @classmethod def parse(cls, source): - """GreaseMonkeyScript factory. + """GreasemonkeyScript factory. - Takes a userscript source and returns a GreaseMonkeyScript. - Parses the greasemonkey metadata block, if present, to fill out + Takes a userscript source and returns a GreasemonkeyScript. + Parses the Greasemonkey metadata block, if present, to fill out attributes. """ matches = re.split(cls.HEADER_REGEX, source, maxsplit=2) try: - _, props, _code = matches + _head, props, _code = matches except ValueError: props = "" script = cls(re.findall(cls.PROPS_REGEX, props), source) @@ -90,9 +91,9 @@ class GreasemonkeyScript: return script def code(self): - """Return the processed javascript code of this script. + """Return the processed JavaScript code of this script. - Adorns the source code with GM_* methods for greasemonkey + Adorns the source code with GM_* methods for Greasemonkey compatibility and wraps it in an IFFE to hide it within a lexical scope. Note that this means line numbers in your browser's debugger/inspector will not match up to the line @@ -118,6 +119,7 @@ class GreasemonkeyScript: @attr.s class MatchingScripts(object): + """All userscripts registered to run on a particular url.""" url = attr.ib() @@ -128,11 +130,11 @@ class MatchingScripts(object): class GreasemonkeyManager(QObject): - """Manager of userscripts and a greasemonkey compatible environment. + """Manager of userscripts and a Greasemonkey compatible environment. Signals: scripts_reloaded: Emitted when scripts are reloaded from disk. - Any any cached or already-injected scripts should be + Any cached or already-injected scripts should be considered obselete. """ @@ -151,8 +153,8 @@ class GreasemonkeyManager(QObject): def load_scripts(self): """Re-read Greasemonkey scripts from disk. - The scripts are read from a 'greasemonkey' subdirectory in qutebrowser's - data directory (see `:version`). + The scripts are read from a 'greasemonkey' subdirectory in + qutebrowser's data directory (see `:version`). """ self._run_start = [] self._run_end = [] diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index dade671c8..c2c388993 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -245,10 +245,10 @@ def _init_profiles(): def inject_userscripts(): - """Register user javascript files with the global profiles.""" - # The greasemonkey metadata block support in qtwebengine only starts at 5.8 - # Otherwise have to handle injecting the scripts into the page at very - # early load, probs same place in view as the enableJS check. + """Register user JavaScript files with the global profiles.""" + # The Greasemonkey metadata block support in QtWebEngine only starts at + # Qt 5.8. With 5.7.1, we need to inject the scripts ourselves in response + # to urlChanged. if not qtutils.version_check('5.8'): return @@ -256,10 +256,7 @@ def inject_userscripts(): # just get replaced by new gm scripts like if we were injecting them # ourselves so we need to remove all gm scripts, while not removing # any other stuff that might have been added. Like the one for - # stylsheets. - # Could either use a different world for gm scripts, check for gm metadata - # values (would mean no non-gm userscripts), or check the code for - # _qute_script_id + # stylesheets. for profile in [default_profile, private_profile]: scripts = profile.scripts() for script in scripts.toList(): diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index af4476d4a..0668e3aa5 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -308,7 +308,7 @@ class WebEnginePage(QWebEnginePage): def _inject_userjs(self, url): """Inject userscripts registered for `url` into the current page.""" if qtutils.version_check('5.8'): - # Handled in webenginetab with the builtin greasemonkey + # Handled in webenginetab with the builtin Greasemonkey # support. return diff --git a/qutebrowser/browser/webkit/webpage.py b/qutebrowser/browser/webkit/webpage.py index 02aa270d7..89407fcdf 100644 --- a/qutebrowser/browser/webkit/webpage.py +++ b/qutebrowser/browser/webkit/webpage.py @@ -95,7 +95,7 @@ class BrowserPage(QWebPage): """Connect userjs related signals to `frame`. Connect the signals used as triggers for injecting user - javascripts into the passed QWebFrame. + JavaScripts into the passed QWebFrame. """ log.greasemonkey.debug("Connecting to frame {} ({})" .format(frame, frame.url().toDisplayString())) @@ -299,7 +299,7 @@ class BrowserPage(QWebPage): self.error_occurred = False def _inject_userjs(self, frame): - """Inject user javascripts into the page. + """Inject user JavaScripts into the page. Args: frame: The QWebFrame to inject the user scripts into. diff --git a/qutebrowser/javascript/greasemonkey_wrapper.js b/qutebrowser/javascript/greasemonkey_wrapper.js index e86991040..2d36220dc 100644 --- a/qutebrowser/javascript/greasemonkey_wrapper.js +++ b/qutebrowser/javascript/greasemonkey_wrapper.js @@ -1,5 +1,5 @@ -(function () { - const _qute_script_id = "__gm_"+{{ scriptName | tojson }}; +(function() { + const _qute_script_id = "__gm_" + {{ scriptName | tojson }}; function GM_log(text) { console.log(text); @@ -10,7 +10,8 @@ 'scriptMetaStr': {{ scriptMeta | tojson }}, 'scriptWillUpdate': false, 'version': "0.0.1", - 'scriptHandler': 'Tampermonkey' // so scripts don't expect exportFunction + // so scripts don't expect exportFunction + 'scriptHandler': 'Tampermonkey', }; function checkKey(key, funcName) { @@ -40,7 +41,7 @@ } function GM_listValues() { - let keys = []; + const keys = []; for (let i = 0; i < localStorage.length; i++) { if (localStorage.key(i).startsWith(_qute_script_id)) { keys.push(localStorage.key(i).slice(_qute_script_id.length)); @@ -59,28 +60,28 @@ details.method = details.method ? details.method.toUpperCase() : "GET"; if (!details.url) { - throw ("GM_xmlhttpRequest requires an URL."); + throw new Error("GM_xmlhttpRequest requires an URL."); } // build XMLHttpRequest object - let oXhr = new XMLHttpRequest(); + const oXhr = new XMLHttpRequest(); // run it if ("onreadystatechange" in details) { - oXhr.onreadystatechange = function () { + oXhr.onreadystatechange = function() { details.onreadystatechange(oXhr); }; } if ("onload" in details) { - oXhr.onload = function () { details.onload(oXhr) }; + oXhr.onload = function() { details.onload(oXhr); }; } if ("onerror" in details) { - oXhr.onerror = function () { details.onerror(oXhr) }; + oXhr.onerror = function () { details.onerror(oXhr); }; } oXhr.open(details.method, details.url, true); if ("headers" in details) { - for (let header in details.headers) { + for (const header in details.headers) { oXhr.setRequestHeader(header, details.headers[header]); } } @@ -93,26 +94,25 @@ } function GM_addStyle(/* String */ styles) { - let oStyle = document.createElement("style"); + const oStyle = document.createElement("style"); oStyle.setAttribute("type", "text/css"); oStyle.appendChild(document.createTextNode(styles)); - let head = document.getElementsByTagName("head")[0]; + const head = document.getElementsByTagName("head")[0]; if (head === undefined) { - document.onreadystatechange = function () { - if (document.readyState == "interactive") { + document.onreadystatechange = function() { + if (document.readyState === "interactive") { document.getElementsByTagName("head")[0].appendChild(oStyle); } - } - } - else { + }; + } else { head.appendChild(oStyle); } } const unsafeWindow = window; - //====== The actual user script source ======// + // ====== The actual user script source ====== // {{ scriptSource }} - //====== End User Script ======// + // ====== End User Script ====== // })(); diff --git a/tests/end2end/features/javascript.feature b/tests/end2end/features/javascript.feature index 944d2606d..3ccd50efb 100644 --- a/tests/end2end/features/javascript.feature +++ b/tests/end2end/features/javascript.feature @@ -124,8 +124,8 @@ Feature: Javascript stuff And I run :tab-next Then the window sizes should be the same - Scenario: Have a greasemonkey script run at page start - When I have a greasemonkey file saved for document-start with noframes unset + Scenario: Have a GreaseMonkey script run at page start + When I have a GreaseMonkey file saved for document-start with noframes unset And I run :greasemonkey-reload And I open data/hints/iframe.html # This second reload is required in webengine < 5.8 for scripts @@ -133,15 +133,15 @@ Feature: Javascript stuff And I run :reload Then the javascript message "Script is running on /data/hints/iframe.html" should be logged - Scenario: Have a greasemonkey script running on frames - When I have a greasemonkey file saved for document-end with noframes unset + Scenario: Have a GreaseMonkey script running on frames + When I have a GreaseMonkey file saved for document-end with noframes unset And I run :greasemonkey-reload And I open data/hints/iframe.html Then the javascript message "Script is running on /data/hints/html/wrapped.html" should be logged @flaky - Scenario: Have a greasemonkey script running on noframes - When I have a greasemonkey file saved for document-end with noframes set + Scenario: Have a GreaseMonkey script running on noframes + When I have a GreaseMonkey file saved for document-end with noframes set And I run :greasemonkey-reload And I open data/hints/iframe.html Then the javascript message "Script is running on /data/hints/html/wrapped.html" should not be logged diff --git a/tests/end2end/features/test_javascript_bdd.py b/tests/end2end/features/test_javascript_bdd.py index 16896d4b5..8f69ef6d4 100644 --- a/tests/end2end/features/test_javascript_bdd.py +++ b/tests/end2end/features/test_javascript_bdd.py @@ -35,7 +35,7 @@ def check_window_sizes(quteproc): test_gm_script = r""" // ==UserScript== -// @name Qutebrowser test userscript +// @name qutebrowser test userscript // @namespace invalid.org // @include http://localhost:*/data/hints/iframe.html // @include http://localhost:*/data/hints/html/wrapped.html @@ -47,7 +47,7 @@ console.log("Script is running on " + window.location.pathname); """ -@bdd.when(bdd.parsers.parse("I have a greasemonkey file saved for {stage} " +@bdd.when(bdd.parsers.parse("I have a GreaseMonkey file saved for {stage} " "with noframes {frameset}")) def create_greasemonkey_file(quteproc, stage, frameset): script_path = os.path.join(quteproc.basedir, 'data', 'greasemonkey') diff --git a/tests/unit/javascript/test_greasemonkey.py b/tests/unit/javascript/test_greasemonkey.py index b0ba64bdf..670be416d 100644 --- a/tests/unit/javascript/test_greasemonkey.py +++ b/tests/unit/javascript/test_greasemonkey.py @@ -28,7 +28,7 @@ from qutebrowser.browser import greasemonkey test_gm_script = """ // ==UserScript== -// @name Qutebrowser test userscript +// @name qutebrowser test userscript // @namespace invalid.org // @include http://localhost:*/data/title.html // @match http://trolol* @@ -58,7 +58,7 @@ def test_all(): gm_manager = greasemonkey.GreasemonkeyManager() assert (gm_manager.all_scripts()[0].name == - "Qutebrowser test userscript") + "qutebrowser test userscript") @pytest.mark.parametrize("url, expected_matches", [ @@ -70,7 +70,7 @@ def test_all(): ('https://badhost.xxx/', 0), ]) def test_get_scripts_by_url(url, expected_matches): - """Check greasemonkey include/exclude rules work.""" + """Check Greasemonkey include/exclude rules work.""" save_script(test_gm_script, 'test.user.js') gm_manager = greasemonkey.GreasemonkeyManager() From a37ecc353cb29d5555dab019e3082a7d82d7eac7 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 19:53:58 +0100 Subject: [PATCH 120/322] Simplify for loop --- qutebrowser/browser/webengine/webenginesettings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index c2c388993..35eb4b916 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -257,6 +257,7 @@ def inject_userscripts(): # ourselves so we need to remove all gm scripts, while not removing # any other stuff that might have been added. Like the one for # stylesheets. + greasemonkey = objreg.get('greasemonkey') for profile in [default_profile, private_profile]: scripts = profile.scripts() for script in scripts.toList(): @@ -265,9 +266,7 @@ def inject_userscripts(): .format(script.name())) scripts.remove(script) - for profile in [default_profile, private_profile]: - scripts = profile.scripts() - greasemonkey = objreg.get('greasemonkey') + # Then add the new scripts. for script in greasemonkey.all_scripts(): new_script = QWebEngineScript() new_script.setWorldId(QWebEngineScript.MainWorld) From 6aafe02320d539943967896cc4901c2eafcb2f47 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 19:56:44 +0100 Subject: [PATCH 121/322] Make sure scripts are removed correctly --- qutebrowser/browser/webengine/webenginesettings.py | 5 +++-- qutebrowser/browser/webengine/webview.py | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index 35eb4b916..adc7a1034 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -262,9 +262,10 @@ def inject_userscripts(): scripts = profile.scripts() for script in scripts.toList(): if script.name().startswith("GM-"): - log.greasemonkey.debug('removing script: {}' + log.greasemonkey.debug('Removing script: {}' .format(script.name())) - scripts.remove(script) + removed = scripts.remove(script) + assert removed, script.name() # Then add the new scripts. for script in greasemonkey.all_scripts(): diff --git a/qutebrowser/browser/webengine/webview.py b/qutebrowser/browser/webengine/webview.py index 0668e3aa5..b313fc36c 100644 --- a/qutebrowser/browser/webengine/webview.py +++ b/qutebrowser/browser/webengine/webview.py @@ -319,9 +319,10 @@ class WebEnginePage(QWebEnginePage): scripts = self.scripts() for script in scripts.toList(): if script.name().startswith("GM-"): - really_removed = scripts.remove(script) - log.greasemonkey.debug("Removing ({}) script: {}" - .format(really_removed, script.name())) + log.greasemonkey.debug("Removing script: {}" + .format(script.name())) + removed = scripts.remove(script) + assert removed, script.name() def _add_script(script, injection_point): new_script = QWebEngineScript() From d6039a0e348651fd4b482b325b4b3ed3c5e146b5 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 20:30:46 +0100 Subject: [PATCH 122/322] Fix markers for editor test --- tests/end2end/features/editor.feature | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/end2end/features/editor.feature b/tests/end2end/features/editor.feature index 3e1be47bc..15da4a6cd 100644 --- a/tests/end2end/features/editor.feature +++ b/tests/end2end/features/editor.feature @@ -116,9 +116,8 @@ Feature: Opening external editors Then the javascript message "text: foobar" should be logged # Could not get signals working on Windows - @posix # There's no guarantee that the tab gets deleted... - @flaky + @posix @flaky Scenario: Spawning an editor and closing the tab When I set up a fake editor that waits And I open data/editor.html From eb90f9835fa2e159842ae16072b80109b08a977c Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 20:54:14 +0100 Subject: [PATCH 123/322] Mark qute://settings test as flaky --- tests/end2end/features/qutescheme.feature | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/end2end/features/qutescheme.feature b/tests/end2end/features/qutescheme.feature index 51db7f767..1f13a8ac1 100644 --- a/tests/end2end/features/qutescheme.feature +++ b/tests/end2end/features/qutescheme.feature @@ -113,6 +113,8 @@ Feature: Special qute:// pages And I wait for "Config option changed: ignore_case *" in the log Then the option ignore_case should be set to always + # Sometimes, an unrelated value gets set + @flaky Scenario: Focusing input fields in qute://settings and entering invalid value When I open qute://settings # scroll to the right - the table does not fit in the default screen From 04d200452817e58d357627ca87aabe6ef7e6b044 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 20:58:14 +0100 Subject: [PATCH 124/322] tox: Fix eslint environment We need to set basepython there to not get InterpreterNotFound --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index e59e8b660..5b8bc05b5 100644 --- a/tox.ini +++ b/tox.ini @@ -181,6 +181,7 @@ commands = [testenv:eslint] # This is duplicated in travis_run.sh for Travis CI because we can't get tox in # the JavaScript environment easily. +basepython = python3 deps = whitelist_externals = eslint changedir = {toxinidir}/qutebrowser/javascript From 30b25da2735e597104f9f6b4fde6842261fcac86 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 6 Dec 2017 13:09:44 -0700 Subject: [PATCH 125/322] Added protocol key to field --- qutebrowser/config/configdata.yml | 4 ++++ qutebrowser/mainwindow/tabwidget.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 08c854ed3..efcc07b35 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1292,6 +1292,7 @@ tabs.title.format: - host - private - current_url + - protocol none_ok: true desc: | Format to use for the tab title. @@ -1308,6 +1309,7 @@ tabs.title.format: * `{backend}`: Either ''webkit'' or ''webengine'' * `{private}` : Indicates when private mode is enabled. * `{current_url}` : URL of the current web page. + * `{protocol}` : Internet Protocol of the current web page. tabs.title.format_pinned: default: '{index}' @@ -1324,6 +1326,7 @@ tabs.title.format_pinned: - host - private - current_url + - protocol none_ok: true desc: Format to use for the tab title for pinned tabs. The same placeholders like for `tabs.title.format` are defined. @@ -1467,6 +1470,7 @@ window.title_format: - backend - private - current_url + - protocol default: '{perc}{title}{title_sep}qutebrowser' desc: | Format to use for the window title. The same placeholders like for diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index b14466337..31615cccc 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -181,6 +181,11 @@ class TabWidget(QTabWidget): except qtutils.QtValueError: fields['current_url'] = '' + try: + fields['protocol'] = self.tab_url(idx).url(options = QUrl.RemovePath) + except qtutils.QtValueError: + fields['protocol']= '' + y = tab.scroller.pos_perc()[1] if y is None: scroll_pos = '???' From f033b228b1d94f7dbba54e552bdf7eb2870a9dd0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 6 Dec 2017 20:22:03 +0100 Subject: [PATCH 126/322] Use py.path.local in save_script --- tests/unit/javascript/test_greasemonkey.py | 24 +++++++++------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/unit/javascript/test_greasemonkey.py b/tests/unit/javascript/test_greasemonkey.py index 670be416d..0f5fe476c 100644 --- a/tests/unit/javascript/test_greasemonkey.py +++ b/tests/unit/javascript/test_greasemonkey.py @@ -18,10 +18,10 @@ """Tests for qutebrowser.browser.greasemonkey.""" -import os import logging import pytest +import py.path # pylint: disable=no-name-in-module from PyQt5.QtCore import QUrl from qutebrowser.browser import greasemonkey @@ -41,20 +41,16 @@ console.log("Script is running."); pytestmark = pytest.mark.usefixtures('data_tmpdir') -def save_script(script_text, filename): - script_path = greasemonkey._scripts_dir() - try: - os.mkdir(script_path) - except FileExistsError: - pass - file_path = os.path.join(script_path, filename) - with open(file_path, 'w', encoding='utf-8') as f: - f.write(script_text) +def _save_script(script_text, filename): + # pylint: disable=no-member + file_path = py.path.local(greasemonkey._scripts_dir()) / filename + # pylint: enable=no-member + file_path.write_text(script_text, encoding='utf-8', ensure=True) def test_all(): """Test that a script gets read from file, parsed and returned.""" - save_script(test_gm_script, 'test.user.js') + _save_script(test_gm_script, 'test.user.js') gm_manager = greasemonkey.GreasemonkeyManager() assert (gm_manager.all_scripts()[0].name == @@ -71,7 +67,7 @@ def test_all(): ]) def test_get_scripts_by_url(url, expected_matches): """Check Greasemonkey include/exclude rules work.""" - save_script(test_gm_script, 'test.user.js') + _save_script(test_gm_script, 'test.user.js') gm_manager = greasemonkey.GreasemonkeyManager() scripts = gm_manager.scripts_for(QUrl(url)) @@ -81,7 +77,7 @@ def test_get_scripts_by_url(url, expected_matches): def test_no_metadata(caplog): """Run on all sites at document-end is the default.""" - save_script("var nothing = true;\n", 'nothing.user.js') + _save_script("var nothing = true;\n", 'nothing.user.js') with caplog.at_level(logging.WARNING): gm_manager = greasemonkey.GreasemonkeyManager() @@ -93,7 +89,7 @@ def test_no_metadata(caplog): def test_bad_scheme(caplog): """qute:// isn't in the list of allowed schemes.""" - save_script("var nothing = true;\n", 'nothing.user.js') + _save_script("var nothing = true;\n", 'nothing.user.js') with caplog.at_level(logging.WARNING): gm_manager = greasemonkey.GreasemonkeyManager() From 94809032a46dddbce62f279d0d9e5b85e50eb52b Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 6 Dec 2017 13:24:27 -0700 Subject: [PATCH 127/322] field[protocol] gives the right protocol] --- qutebrowser/mainwindow/tabwidget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 31615cccc..988455a08 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -182,7 +182,7 @@ class TabWidget(QTabWidget): fields['current_url'] = '' try: - fields['protocol'] = self.tab_url(idx).url(options = QUrl.RemovePath) + fields['protocol'] = self.tab_url(idx).scheme() except qtutils.QtValueError: fields['protocol']= '' From 02b24e8dfb4e7a2e65c363ba4e82708386e8e81a Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Wed, 6 Dec 2017 21:35:09 -0700 Subject: [PATCH 128/322] Update tabwidget.py --- qutebrowser/mainwindow/tabwidget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 988455a08..01b9f0728 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -184,7 +184,7 @@ class TabWidget(QTabWidget): try: fields['protocol'] = self.tab_url(idx).scheme() except qtutils.QtValueError: - fields['protocol']= '' + fields['protocol'] = '' y = tab.scroller.pos_perc()[1] if y is None: From 4d13941290f28dfe5de2ec3be3238fba41fb263b Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Wed, 6 Dec 2017 23:57:19 -0700 Subject: [PATCH 129/322] added the scheme field to FakeURL --- tests/helpers/stubs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 878c9e166..cb3c85896 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -131,11 +131,12 @@ class FakeUrl: """QUrl stub which provides .path(), isValid() and host().""" - def __init__(self, path=None, valid=True, host=None, url=None): + def __init__(self, path=None, valid=True, host=None, url=None, scheme = None): self.path = mock.Mock(return_value=path) self.isValid = mock.Mock(returl_value=valid) self.host = mock.Mock(returl_value=host) self.url = mock.Mock(return_value=url) + self.scheme = mock.Mock(return_value=scheme) class FakeNetworkReply: From d4cadcc62ea4ecd65b88aec90a27ceef8ed9fa86 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Thu, 7 Dec 2017 08:17:15 +0100 Subject: [PATCH 130/322] Add comment about @run-at [ci skip] --- qutebrowser/browser/webengine/webenginesettings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qutebrowser/browser/webengine/webenginesettings.py b/qutebrowser/browser/webengine/webenginesettings.py index adc7a1034..f8b54e065 100644 --- a/qutebrowser/browser/webengine/webenginesettings.py +++ b/qutebrowser/browser/webengine/webenginesettings.py @@ -269,6 +269,8 @@ def inject_userscripts(): # Then add the new scripts. for script in greasemonkey.all_scripts(): + # @run-at (and @include/@exclude/@match) is parsed by + # QWebEngineScript. new_script = QWebEngineScript() new_script.setWorldId(QWebEngineScript.MainWorld) new_script.setSourceCode(script.code()) From 20ac618752a6e33f7cd8ed51e6e8da8213eee29b Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Thu, 7 Dec 2017 02:04:02 -0700 Subject: [PATCH 131/322] Simplified code in get_tab_fields changed self.tab_url(idx) to url in get_tab_fields() --- qutebrowser/mainwindow/tabwidget.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 01b9f0728..9dc6cd9bf 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -165,6 +165,8 @@ class TabWidget(QTabWidget): fields['perc_raw'] = tab.progress() fields['backend'] = objects.backend.name fields['private'] = ' [Private Mode] ' if tab.private else '' + + url = self.tab_url(idx) if tab.load_status() == usertypes.LoadStatus.loading: fields['perc'] = '[{}%] '.format(tab.progress()) @@ -172,17 +174,17 @@ class TabWidget(QTabWidget): fields['perc'] = '' try: - fields['host'] = self.tab_url(idx).host() + fields['host'] = url.host() except qtutils.QtValueError: fields['host'] = '' try: - fields['current_url'] = self.tab_url(idx).url() + fields['current_url'] = url.url() except qtutils.QtValueError: fields['current_url'] = '' try: - fields['protocol'] = self.tab_url(idx).scheme() + fields['protocol'] = url.scheme() except qtutils.QtValueError: fields['protocol'] = '' From d1a00eb93446e501e97871485738b903668a89a5 Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Thu, 7 Dec 2017 02:35:34 -0700 Subject: [PATCH 132/322] Clarity on protocol field --- qutebrowser/config/configdata.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index efcc07b35..7c94d4701 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1307,9 +1307,9 @@ tabs.title.format: * `{scroll_pos}`: Page scroll position. * `{host}`: Host of the current web page. * `{backend}`: Either ''webkit'' or ''webengine'' - * `{private}` : Indicates when private mode is enabled. - * `{current_url}` : URL of the current web page. - * `{protocol}` : Internet Protocol of the current web page. + * `{private}`: Indicates when private mode is enabled. + * `{current_url}`: URL of the current web page. + * `{protocol}`: Protocol (http/https/...) of the current web page tabs.title.format_pinned: default: '{index}' From 18609f1a2444cc533c4fabdf046f3da9ff6e3278 Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Thu, 7 Dec 2017 02:36:31 -0700 Subject: [PATCH 133/322] fixed spacing on FakeURL --- tests/helpers/stubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index cb3c85896..d2c0fd746 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -131,7 +131,7 @@ class FakeUrl: """QUrl stub which provides .path(), isValid() and host().""" - def __init__(self, path=None, valid=True, host=None, url=None, scheme = None): + def __init__(self, path=None, valid=True, host=None, url=None, scheme=None): self.path = mock.Mock(return_value=path) self.isValid = mock.Mock(returl_value=valid) self.host = mock.Mock(returl_value=host) From 25526f00bf959432c2d0d40606169a50173476b3 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 7 Dec 2017 15:47:03 -0700 Subject: [PATCH 134/322] fixed catch error in tabwidget --- qutebrowser/mainwindow/tabwidget.py | 16 +++++----------- tests/helpers/stubs.py | 3 ++- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/qutebrowser/mainwindow/tabwidget.py b/qutebrowser/mainwindow/tabwidget.py index 9dc6cd9bf..28cfac0fb 100644 --- a/qutebrowser/mainwindow/tabwidget.py +++ b/qutebrowser/mainwindow/tabwidget.py @@ -165,8 +165,6 @@ class TabWidget(QTabWidget): fields['perc_raw'] = tab.progress() fields['backend'] = objects.backend.name fields['private'] = ' [Private Mode] ' if tab.private else '' - - url = self.tab_url(idx) if tab.load_status() == usertypes.LoadStatus.loading: fields['perc'] = '[{}%] '.format(tab.progress()) @@ -174,19 +172,15 @@ class TabWidget(QTabWidget): fields['perc'] = '' try: - fields['host'] = url.host() + url = self.tab_url(idx) except qtutils.QtValueError: fields['host'] = '' - - try: - fields['current_url'] = url.url() - except qtutils.QtValueError: fields['current_url'] = '' - - try: - fields['protocol'] = url.scheme() - except qtutils.QtValueError: fields['protocol'] = '' + else: + fields['host'] = url.host() + fields['current_url'] = url.toDisplayString() + fields['protocol'] = url.scheme() y = tab.scroller.pos_perc()[1] if y is None: diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index d2c0fd746..ede073130 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -131,7 +131,8 @@ class FakeUrl: """QUrl stub which provides .path(), isValid() and host().""" - def __init__(self, path=None, valid=True, host=None, url=None, scheme=None): + def __init__(self, path=None, valid=True, host=None, url=None, + scheme=None): self.path = mock.Mock(return_value=path) self.isValid = mock.Mock(returl_value=valid) self.host = mock.Mock(returl_value=host) From 9685eb36b69e091cc99652f4cfc4f73d331d1aab Mon Sep 17 00:00:00 2001 From: evanlee123 <31551958+evanlee123@users.noreply.github.com> Date: Thu, 7 Dec 2017 16:30:34 -0700 Subject: [PATCH 135/322] Changed FakeUrl's url command to toDisplayString --- tests/helpers/stubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index ede073130..0fd2ba0ed 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -136,7 +136,7 @@ class FakeUrl: self.path = mock.Mock(return_value=path) self.isValid = mock.Mock(returl_value=valid) self.host = mock.Mock(returl_value=host) - self.url = mock.Mock(return_value=url) + self.toDisplayString = mock.Mock(return_value=url) self.scheme = mock.Mock(return_value=scheme) From 9f9311840a2a46b25288d5ed2f432bec27802a9a Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Fri, 8 Dec 2017 16:44:53 +0000 Subject: [PATCH 136/322] Add --output-to-tab flag for :spawn. This puts the exit status, stdout, and stderr in a new tab. --- doc/help/commands.asciidoc | 3 ++- qutebrowser/browser/commands.py | 6 ++++-- qutebrowser/browser/qutescheme.py | 8 ++++++++ qutebrowser/misc/guiprocess.py | 33 ++++++++++++++++++++++++++++--- 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 5d026bfca..2487b8c85 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -1148,7 +1148,7 @@ Set a mark at the current scroll position in the current tab. [[spawn]] === spawn -Syntax: +:spawn [*--userscript*] [*--verbose*] [*--detach*] 'cmdline'+ +Syntax: +:spawn [*--userscript*] [*--verbose*] [*--output-to-tab*] [*--detach*] 'cmdline'+ Spawn a command in a shell. @@ -1163,6 +1163,7 @@ Spawn a command in a shell. - `/usr/share/qutebrowser/userscripts` * +*-v*+, +*--verbose*+: Show notifications when the command started/exited. +* +*-v*+, +*--output-to-tab*+: Show stderr, stdout, and exit status in a new tab. * +*-d*+, +*--detach*+: Whether the command should be detached from qutebrowser. ==== note diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index bced4daf4..8c623aa37 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1177,7 +1177,8 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0, no_replace_variables=True) - def spawn(self, cmdline, userscript=False, verbose=False, detach=False): + def spawn(self, cmdline, userscript=False, verbose=False, + output_to_tab=False, detach=False): """Spawn a command in a shell. Args: @@ -1208,7 +1209,8 @@ class CommandDispatcher: else: cmd = os.path.expanduser(cmd) proc = guiprocess.GUIProcess(what='command', verbose=verbose, - parent=self._tabbed_browser) + parent=self._tabbed_browser, + output_to_tab=output_to_tab) if detach: proc.start_detached(cmd, args) else: diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index e6262a007..3bd2c66a4 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -42,6 +42,7 @@ from qutebrowser.misc import objects pyeval_output = ":pyeval was never called" +spawn_output = ":spawn was never called" _HANDLERS = {} @@ -268,6 +269,13 @@ def qute_pyeval(_url): return 'text/html', html +@add_handler('spawn_output') +def qute_spawn_output(_url): + """Handler for qute://spawn_output.""" + html = jinja.render('pre.html', title='spawn output', content=spawn_output) + return 'text/html', html + + @add_handler('version') @add_handler('verizon') def qute_version(_url): diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index 1adf6817e..3c2057864 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -22,9 +22,11 @@ import shlex from PyQt5.QtCore import (pyqtSlot, pyqtSignal, QObject, QProcess, - QProcessEnvironment) + QProcessEnvironment, QUrl) -from qutebrowser.utils import message, log +from qutebrowser.utils import message, log, objreg + +from qutebrowser.browser import qutescheme # A mapping of QProcess::ErrorCode's to human-readable strings. @@ -62,10 +64,11 @@ class GUIProcess(QObject): started = pyqtSignal() def __init__(self, what, *, verbose=False, additional_env=None, - parent=None): + parent=None, output_to_tab=False): super().__init__(parent) self._what = what self.verbose = verbose + self.output_to_tab = output_to_tab self._started = False self.cmd = None self.args = None @@ -96,6 +99,30 @@ class GUIProcess(QObject): self._started = False log.procs.debug("Process finished with code {}, status {}.".format( code, status)) + + stdout = None + stderr = None + + if self.output_to_tab: + stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') + stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') + + spawn_log = "" + + spawn_log += "Process finished with code {},status {}.".format( + code, status) + + if stdout: + spawn_log += "\nProcess stdout:\n" + stdout.strip() + if stderr: + spawn_log += "\nProcess stderr:\n" + stderr.strip() + + qutescheme.spawn_output = spawn_log + + tabbed_browser = objreg.get('tabbed-browser', scope='window', + window='last-focused') + tabbed_browser.openurl(QUrl('qute://spawn_output'), newtab=True) + if status == QProcess.CrashExit: message.error("{} crashed!".format(self._what.capitalize())) elif status == QProcess.NormalExit and code == 0: From 9f8dbe95e47e5ac9440b89e31c72edb79857ee0a Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Fri, 8 Dec 2017 19:00:46 +0000 Subject: [PATCH 137/322] Code review changes. This fixes the following problems found in a review: 1. Manual modification of the asciidoc has been undone. 2. --output-to-tab has been renamed to the less verbose --output. 3. spawn_output has been changed to spawn-output in the url. 4. Erroneous newline in imports has been removed. 5. output in guiprocess.py has been marked private. 6. If there is no output for either stderr or stdout, say so. 7. Missing space in a text line was added. 8. Redundant initialising of an empty string removed. --- doc/help/commands.asciidoc | 4 ++-- qutebrowser/browser/commands.py | 5 +++-- qutebrowser/browser/qutescheme.py | 4 ++-- qutebrowser/misc/guiprocess.py | 24 ++++++++++-------------- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/doc/help/commands.asciidoc b/doc/help/commands.asciidoc index 2487b8c85..3da81a391 100644 --- a/doc/help/commands.asciidoc +++ b/doc/help/commands.asciidoc @@ -1148,7 +1148,7 @@ Set a mark at the current scroll position in the current tab. [[spawn]] === spawn -Syntax: +:spawn [*--userscript*] [*--verbose*] [*--output-to-tab*] [*--detach*] 'cmdline'+ +Syntax: +:spawn [*--userscript*] [*--verbose*] [*--output*] [*--detach*] 'cmdline'+ Spawn a command in a shell. @@ -1163,7 +1163,7 @@ Spawn a command in a shell. - `/usr/share/qutebrowser/userscripts` * +*-v*+, +*--verbose*+: Show notifications when the command started/exited. -* +*-v*+, +*--output-to-tab*+: Show stderr, stdout, and exit status in a new tab. +* +*-o*+, +*--output*+: Whether the output should be shown in a new tab. * +*-d*+, +*--detach*+: Whether the command should be detached from qutebrowser. ==== note diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 8c623aa37..7ad73708a 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1178,7 +1178,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0, no_replace_variables=True) def spawn(self, cmdline, userscript=False, verbose=False, - output_to_tab=False, detach=False): + output=False, detach=False): """Spawn a command in a shell. Args: @@ -1189,6 +1189,7 @@ class CommandDispatcher: (or `$XDG_DATA_DIR`) - `/usr/share/qutebrowser/userscripts` verbose: Show notifications when the command started/exited. + output: Whether the output should be shown in a new tab. detach: Whether the command should be detached from qutebrowser. cmdline: The commandline to execute. """ @@ -1210,7 +1211,7 @@ class CommandDispatcher: cmd = os.path.expanduser(cmd) proc = guiprocess.GUIProcess(what='command', verbose=verbose, parent=self._tabbed_browser, - output_to_tab=output_to_tab) + output=output) if detach: proc.start_detached(cmd, args) else: diff --git a/qutebrowser/browser/qutescheme.py b/qutebrowser/browser/qutescheme.py index 3bd2c66a4..32bc5806a 100644 --- a/qutebrowser/browser/qutescheme.py +++ b/qutebrowser/browser/qutescheme.py @@ -269,9 +269,9 @@ def qute_pyeval(_url): return 'text/html', html -@add_handler('spawn_output') +@add_handler('spawn-output') def qute_spawn_output(_url): - """Handler for qute://spawn_output.""" + """Handler for qute://spawn-output.""" html = jinja.render('pre.html', title='spawn output', content=spawn_output) return 'text/html', html diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index 3c2057864..32a4fbf62 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -25,7 +25,6 @@ from PyQt5.QtCore import (pyqtSlot, pyqtSignal, QObject, QProcess, QProcessEnvironment, QUrl) from qutebrowser.utils import message, log, objreg - from qutebrowser.browser import qutescheme # A mapping of QProcess::ErrorCode's to human-readable strings. @@ -50,6 +49,7 @@ class GUIProcess(QObject): cmd: The command which was started. args: A list of arguments which gets passed. verbose: Whether to show more messages. + _output: Whether to show the output in a new tab. _started: Whether the underlying process is started. _proc: The underlying QProcess. _what: What kind of thing is spawned (process/editor/userscript/...). @@ -64,11 +64,11 @@ class GUIProcess(QObject): started = pyqtSignal() def __init__(self, what, *, verbose=False, additional_env=None, - parent=None, output_to_tab=False): + parent=None, output=False): super().__init__(parent) self._what = what self.verbose = verbose - self.output_to_tab = output_to_tab + self._output = output self._started = False self.cmd = None self.args = None @@ -100,28 +100,24 @@ class GUIProcess(QObject): log.procs.debug("Process finished with code {}, status {}.".format( code, status)) - stdout = None - stderr = None - - if self.output_to_tab: + if self._output: stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') - spawn_log = "" + stderr = stderr or "(No output)" + stdout = stdout or "(No output)" - spawn_log += "Process finished with code {},status {}.".format( + spawn_log = "Process finished with code {}, status {}.".format( code, status) - if stdout: - spawn_log += "\nProcess stdout:\n" + stdout.strip() - if stderr: - spawn_log += "\nProcess stderr:\n" + stderr.strip() + spawn_log += "\nProcess stdout:\n" + stdout.strip() + spawn_log += "\nProcess stderr:\n" + stderr.strip() qutescheme.spawn_output = spawn_log tabbed_browser = objreg.get('tabbed-browser', scope='window', window='last-focused') - tabbed_browser.openurl(QUrl('qute://spawn_output'), newtab=True) + tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True) if status == QProcess.CrashExit: message.error("{} crashed!".format(self._what.capitalize())) From 038bb85a67c1115c7091f89c3f83fa55381ea647 Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Sun, 10 Dec 2017 19:12:47 +0000 Subject: [PATCH 138/322] Capture stdout and stderr always for spawn. This change makes it so that stderr and stdout is unconditionally read from for a completed process, and sent to qute://spawn-output. This allows the user to see the results of the previous process, even if they had forgotten to use --output. --- qutebrowser/misc/guiprocess.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index 32a4fbf62..9d0b50be6 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -100,21 +100,18 @@ class GUIProcess(QObject): log.procs.debug("Process finished with code {}, status {}.".format( code, status)) + stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') + stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') + + spawn_log = "Process finished with code {}, status {}.".format( + code, status) + + spawn_log += "\nProcess stdout:\n" + (stdout or "(No output)").strip() + spawn_log += "\nProcess stderr:\n" + (stderr or "(No output)").strip() + + qutescheme.spawn_output = spawn_log + if self._output: - stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') - stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') - - stderr = stderr or "(No output)" - stdout = stdout or "(No output)" - - spawn_log = "Process finished with code {}, status {}.".format( - code, status) - - spawn_log += "\nProcess stdout:\n" + stdout.strip() - spawn_log += "\nProcess stderr:\n" + stderr.strip() - - qutescheme.spawn_output = spawn_log - tabbed_browser = objreg.get('tabbed-browser', scope='window', window='last-focused') tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True) @@ -132,8 +129,6 @@ class GUIProcess(QObject): message.error("{} exited with status {}, see :messages for " "details.".format(self._what.capitalize(), code)) - stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') - stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') if stdout: log.procs.error("Process stdout:\n" + stdout.strip()) if stderr: From d32a4ea99e2d3ff691c14b6ba8ff8ec6d3d4a35a Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Sun, 10 Dec 2017 23:45:43 +0000 Subject: [PATCH 139/322] Seperate _output from guiprocess and keep window opening in spawn. This removes the extraneous variable, and makes testing easier. --- qutebrowser/browser/commands.py | 8 ++++++-- qutebrowser/misc/guiprocess.py | 33 ++++++++++++++++----------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 7ad73708a..495d48414 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1210,13 +1210,17 @@ class CommandDispatcher: else: cmd = os.path.expanduser(cmd) proc = guiprocess.GUIProcess(what='command', verbose=verbose, - parent=self._tabbed_browser, - output=output) + parent=self._tabbed_browser) if detach: proc.start_detached(cmd, args) else: proc.start(cmd, args) + if output: + tabbed_browser = objreg.get('tabbed-browser', scope='window', + window='last-focused') + tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True) + @cmdutils.register(instance='command-dispatcher', scope='window') def home(self): """Open main startpage in current tab.""" diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index 9d0b50be6..80803f53f 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -22,9 +22,9 @@ import shlex from PyQt5.QtCore import (pyqtSlot, pyqtSignal, QObject, QProcess, - QProcessEnvironment, QUrl) + QProcessEnvironment) -from qutebrowser.utils import message, log, objreg +from qutebrowser.utils import message, log from qutebrowser.browser import qutescheme # A mapping of QProcess::ErrorCode's to human-readable strings. @@ -49,7 +49,6 @@ class GUIProcess(QObject): cmd: The command which was started. args: A list of arguments which gets passed. verbose: Whether to show more messages. - _output: Whether to show the output in a new tab. _started: Whether the underlying process is started. _proc: The underlying QProcess. _what: What kind of thing is spawned (process/editor/userscript/...). @@ -64,11 +63,10 @@ class GUIProcess(QObject): started = pyqtSignal() def __init__(self, what, *, verbose=False, additional_env=None, - parent=None, output=False): + parent=None): super().__init__(parent) self._what = what self.verbose = verbose - self._output = output self._started = False self.cmd = None self.args = None @@ -103,18 +101,8 @@ class GUIProcess(QObject): stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') - spawn_log = "Process finished with code {}, status {}.".format( - code, status) - - spawn_log += "\nProcess stdout:\n" + (stdout or "(No output)").strip() - spawn_log += "\nProcess stderr:\n" + (stderr or "(No output)").strip() - - qutescheme.spawn_output = spawn_log - - if self._output: - tabbed_browser = objreg.get('tabbed-browser', scope='window', - window='last-focused') - tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True) + qutescheme.spawn_output = self.spawn_format(code, status, + stdout, stderr) if status == QProcess.CrashExit: message.error("{} crashed!".format(self._what.capitalize())) @@ -134,6 +122,17 @@ class GUIProcess(QObject): if stderr: log.procs.error("Process stderr:\n" + stderr.strip()) + def spawn_format(self, code=0, status=0, stdout="", stderr=""): + """Produce a formatted string for spawn output.""" + stdout = (stdout or "(No output)").strip() + stderr = (stderr or "(No output)").strip() + + spawn_string = ("Process finished with code {}, status {}\n" + "\nProcess stdout:\n {}" + "\nProcess stderr:\n {}").format(code, status, + stdout, stderr) + return spawn_string + @pyqtSlot() def on_started(self): """Called when the process started successfully.""" From 3b10584749c09b44e85ce45233da7d3fe0e870e9 Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Sun, 10 Dec 2017 23:46:35 +0000 Subject: [PATCH 140/322] Update tests to work with the earlier consumption of stdin etc. Note: this adds an element to vulture's whitelist that vulture mistakenly identified as unused. --- scripts/dev/run_vulture.py | 1 + tests/unit/misc/test_guiprocess.py | 30 +++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/scripts/dev/run_vulture.py b/scripts/dev/run_vulture.py index 9d21ad428..657d4b85e 100755 --- a/scripts/dev/run_vulture.py +++ b/scripts/dev/run_vulture.py @@ -82,6 +82,7 @@ def whitelist_generator(): # noqa yield 'qutebrowser.utils.jinja.Loader.get_source' yield 'qutebrowser.utils.log.QtWarningFilter.filter' yield 'qutebrowser.browser.pdfjs.is_available' + yield 'qutebrowser.misc.guiprocess.spawn_output' yield 'QEvent.posted' yield 'log_stack' # from message.py yield 'propagate' # logging.getLogger('...).propagate = False diff --git a/tests/unit/misc/test_guiprocess.py b/tests/unit/misc/test_guiprocess.py index 674c250e5..69ce3812f 100644 --- a/tests/unit/misc/test_guiprocess.py +++ b/tests/unit/misc/test_guiprocess.py @@ -19,7 +19,6 @@ """Tests for qutebrowser.misc.guiprocess.""" -import json import logging import pytest @@ -27,6 +26,7 @@ from PyQt5.QtCore import QProcess, QIODevice from qutebrowser.misc import guiprocess from qutebrowser.utils import usertypes +from qutebrowser.browser import qutescheme @pytest.fixture() @@ -60,7 +60,7 @@ def test_start(proc, qtbot, message_mock, py_proc): proc.start(*argv) assert not message_mock.messages - assert bytes(proc._proc.readAll()).rstrip() == b'test' + assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") def test_start_verbose(proc, qtbot, message_mock, py_proc): @@ -77,7 +77,24 @@ def test_start_verbose(proc, qtbot, message_mock, py_proc): assert msgs[1].level == usertypes.MessageLevel.info assert msgs[0].text.startswith("Executing:") assert msgs[1].text == "Testprocess exited successfully." - assert bytes(proc._proc.readAll()).rstrip() == b'test' + assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") + + +def test_start_output(proc, qtbot, message_mock, py_proc): + """Test starting a process verbosely.""" + proc.verbose = True + + with qtbot.waitSignals([proc.started, proc.finished], timeout=10000, + order='strict'): + argv = py_proc("import sys; print('test'); sys.exit(0)") + proc.start(*argv) + + msgs = message_mock.messages + assert msgs[0].level == usertypes.MessageLevel.info + assert msgs[1].level == usertypes.MessageLevel.info + assert msgs[0].text.startswith("Executing:") + assert msgs[1].text == "Testprocess exited successfully." + assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") def test_start_env(monkeypatch, qtbot, py_proc): @@ -99,10 +116,9 @@ def test_start_env(monkeypatch, qtbot, py_proc): order='strict'): proc.start(*argv) - data = bytes(proc._proc.readAll()).decode('utf-8') - ret_env = json.loads(data) - assert 'QUTEBROWSER_TEST_1' in ret_env - assert 'QUTEBROWSER_TEST_2' in ret_env + data = qutescheme.spawn_output + assert 'QUTEBROWSER_TEST_1' in data + assert 'QUTEBROWSER_TEST_2' in data @pytest.mark.qt_log_ignore('QIODevice::read.*: WriteOnly device') From 2a8b74cbec45d70dd0964384779ba69e1348bd28 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 07:10:17 +0100 Subject: [PATCH 141/322] Get rid of FakeUrl stub We can just use a real QUrl... --- tests/helpers/stubs.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py index 0fd2ba0ed..3f6a23958 100644 --- a/tests/helpers/stubs.py +++ b/tests/helpers/stubs.py @@ -24,7 +24,7 @@ from unittest import mock import attr -from PyQt5.QtCore import pyqtSignal, QPoint, QProcess, QObject +from PyQt5.QtCore import pyqtSignal, QPoint, QProcess, QObject, QUrl from PyQt5.QtNetwork import (QNetworkRequest, QAbstractNetworkCache, QNetworkCacheMetaData) from PyQt5.QtWidgets import QCommonStyle, QLineEdit, QWidget, QTabBar @@ -127,19 +127,6 @@ class FakeQApplication: self.activeWindow = lambda: active_window -class FakeUrl: - - """QUrl stub which provides .path(), isValid() and host().""" - - def __init__(self, path=None, valid=True, host=None, url=None, - scheme=None): - self.path = mock.Mock(return_value=path) - self.isValid = mock.Mock(returl_value=valid) - self.host = mock.Mock(returl_value=host) - self.toDisplayString = mock.Mock(return_value=url) - self.scheme = mock.Mock(return_value=scheme) - - class FakeNetworkReply: """QNetworkReply stub which provides a Content-Disposition header.""" @@ -150,7 +137,7 @@ class FakeNetworkReply: def __init__(self, headers=None, url=None): if url is None: - url = FakeUrl() + url = QUrl() if headers is None: self.headers = {} else: @@ -246,7 +233,7 @@ class FakeWebTab(browsertab.AbstractTab): """Fake AbstractTab to use in tests.""" - def __init__(self, url=FakeUrl(), title='', tab_id=0, *, + def __init__(self, url=QUrl(), title='', tab_id=0, *, scroll_pos_perc=(0, 0), load_status=usertypes.LoadStatus.success, progress=0, can_go_back=None, can_go_forward=None): From 444f0a36dfba2f54a279b0ca8dffd2c601211666 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 07:12:45 +0100 Subject: [PATCH 142/322] Update docs --- doc/changelog.asciidoc | 1 + doc/help/settings.asciidoc | 5 +++-- qutebrowser/config/configdata.yml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 16075db27..4192ee054 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -56,6 +56,7 @@ Added - New `session.lazy_restore` setting which allows to not load pages immediately when restoring a session. - New `hist_importer.py` script to import history from Firefox/Chromium. +- New `{protocol}` replacement for `tabs.title.format` and friends. Changed ~~~~~~~ diff --git a/doc/help/settings.asciidoc b/doc/help/settings.asciidoc index 967f6c4c6..28480486f 100644 --- a/doc/help/settings.asciidoc +++ b/doc/help/settings.asciidoc @@ -2913,8 +2913,9 @@ The following placeholders are defined: * `{scroll_pos}`: Page scroll position. * `{host}`: Host of the current web page. * `{backend}`: Either ''webkit'' or ''webengine'' -* `{private}` : Indicates when private mode is enabled. -* `{current_url}` : URL of the current web page. +* `{private}`: Indicates when private mode is enabled. +* `{current_url}`: URL of the current web page. +* `{protocol}`: Protocol (http/https/...) of the current web page. Type: <> diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index d96718d78..1318e8979 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1317,7 +1317,7 @@ tabs.title.format: * `{backend}`: Either ''webkit'' or ''webengine'' * `{private}`: Indicates when private mode is enabled. * `{current_url}`: URL of the current web page. - * `{protocol}`: Protocol (http/https/...) of the current web page + * `{protocol}`: Protocol (http/https/...) of the current web page. tabs.title.format_pinned: default: '{index}' From f7a94b946f02dae81d32616ed3467ca51314aec1 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 08:12:13 +0100 Subject: [PATCH 143/322] Add changelog for font size change See 22f3fade24dbe6b3bdf737c8395360ca9e6b203b --- doc/changelog.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog.asciidoc b/doc/changelog.asciidoc index 4192ee054..fddb3f6a5 100644 --- a/doc/changelog.asciidoc +++ b/doc/changelog.asciidoc @@ -97,6 +97,7 @@ Changed - Empty categories are now hidden in the `:open` completion. - Search terms for URLs and titles can now be mixed when filtering the completion. +- The default font size for the UI got bumped up from 8pt to 10pt. Fixed ~~~~~ From 6a7d2f4275e18b289cdc41cab6e06984bea2d271 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 09:14:26 +0100 Subject: [PATCH 144/322] Remove dead code QUrl.path() never returns None --- qutebrowser/browser/webkit/http.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qutebrowser/browser/webkit/http.py b/qutebrowser/browser/webkit/http.py index 08cad7a44..7f5173943 100644 --- a/qutebrowser/browser/webkit/http.py +++ b/qutebrowser/browser/webkit/http.py @@ -57,9 +57,7 @@ def parse_content_disposition(reply): is_inline = content_disposition.is_inline() # Then try to get filename from url if not filename: - path = reply.url().path() - if path is not None: - filename = path.rstrip('/') + filename = reply.url().path().rstrip('/') # If that fails as well, use a fallback if not filename: filename = 'qutebrowser-download' From 72d847d687ad00effb73d77e8b0b395f34bdcdf0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 09:36:27 +0100 Subject: [PATCH 145/322] travis: Use newer macOS image --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0ea8218a6..251842d06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ matrix: env: TESTENV=py36-pyqt59-cov - os: osx env: TESTENV=py36 OSX=sierra - osx_image: xcode8.3 + osx_image: xcode9.2 language: generic # https://github.com/qutebrowser/qutebrowser/issues/2013 # - os: osx From 8909e03f1cea998b6e997486e2541bce3d94d71b Mon Sep 17 00:00:00 2001 From: Ryan Roden-Corrent Date: Sun, 10 Dec 2017 15:06:45 -0500 Subject: [PATCH 146/322] Match url completion terms in any order. Perviously, 'foo bar' would match 'foo/bar' but not 'bar/foo'. Now it will match both, using a query with a WHERE clause like: WHERE ((url || title) like '%foo%' AND (url || title) like '%bar%') This does not seem to change the performance benchmark. However, it does create a new query for every character added rather than re-running the same query with different parameters. We could re-use queries if we maintained a list like self._queries=[1_arg_query, 2_arg_query, ...]. However, it isn't clear that such a complexity would be necessary. Resolves #1651. --- qutebrowser/completion/models/histcategory.py | 42 ++++++++++--------- tests/unit/completion/test_histcategory.py | 2 +- tests/unit/completion/test_models.py | 4 +- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/qutebrowser/completion/models/histcategory.py b/qutebrowser/completion/models/histcategory.py index fe89dc79b..03ddb5ff8 100644 --- a/qutebrowser/completion/models/histcategory.py +++ b/qutebrowser/completion/models/histcategory.py @@ -37,21 +37,6 @@ class HistoryCategory(QSqlQueryModel): super().__init__(parent=parent) self.name = "History" - # replace ' in timestamp-format to avoid breaking the query - timestamp_format = config.val.completion.timestamp_format - timefmt = ("strftime('{}', last_atime, 'unixepoch', 'localtime')" - .format(timestamp_format.replace("'", "`"))) - - self._query = sql.Query(' '.join([ - "SELECT url, title, {}".format(timefmt), - "FROM CompletionHistory", - # the incoming pattern will have literal % and _ escaped with '\' - # we need to tell sql to treat '\' as an escape character - "WHERE ((url || title) LIKE :pat escape '\\')", - self._atime_expr(), - "ORDER BY last_atime DESC", - ]), forward_only=False) - # advertise that this model filters by URL and title self.columns_to_filter = [0, 1] self.delete_func = delete_func @@ -86,11 +71,30 @@ class HistoryCategory(QSqlQueryModel): # escape to treat a user input % or _ as a literal, not a wildcard pattern = pattern.replace('%', '\\%') pattern = pattern.replace('_', '\\_') - # treat spaces as wildcards to match any of the typed words - pattern = re.sub(r' +', '%', pattern) - pattern = '%{}%'.format(pattern) + words = ['%{}%'.format(w) for w in pattern.split(' ')] + + wheres = ' AND '.join([ + "(url || title) LIKE :pat{} escape '\\'".format(i) + for i in range(len(words))]) + + # replace ' in timestamp-format to avoid breaking the query + timestamp_format = config.val.completion.timestamp_format + timefmt = ("strftime('{}', last_atime, 'unixepoch', 'localtime')" + .format(timestamp_format.replace("'", "`"))) + + self._query = sql.Query(' '.join([ + "SELECT url, title, {}".format(timefmt), + "FROM CompletionHistory", + # the incoming pattern will have literal % and _ escaped with '\' + # we need to tell sql to treat '\' as an escape character + 'WHERE ({})'.format(wheres), + self._atime_expr(), + "ORDER BY last_atime DESC", + ]), forward_only=False) + with debug.log_time('sql', 'Running completion query'): - self._query.run(pat=pattern) + self._query.run(**{ + 'pat{}'.format(i): w for i, w in enumerate(words)}) self.setQuery(self._query) def removeRows(self, row, _count, _parent=None): diff --git a/tests/unit/completion/test_histcategory.py b/tests/unit/completion/test_histcategory.py index b87eb6ac2..8458c3311 100644 --- a/tests/unit/completion/test_histcategory.py +++ b/tests/unit/completion/test_histcategory.py @@ -61,7 +61,7 @@ def hist(init_sql, config_stub): ('foo bar', [('foo', ''), ('bar foo', ''), ('xfooyybarz', '')], - [('xfooyybarz', '')]), + [('bar foo', ''), ('xfooyybarz', '')]), ('foo%bar', [('foo%bar', ''), ('foo bar', ''), ('foobar', '')], diff --git a/tests/unit/completion/test_models.py b/tests/unit/completion/test_models.py index 8879f3201..c4d224dcc 100644 --- a/tests/unit/completion/test_models.py +++ b/tests/unit/completion/test_models.py @@ -414,12 +414,12 @@ def test_url_completion_no_bookmarks(qtmodeltester, web_history_populated, ('example.com', 'Site Title', 'am', 1), ('example.com', 'Site Title', 'com', 1), ('example.com', 'Site Title', 'ex com', 1), - ('example.com', 'Site Title', 'com ex', 0), + ('example.com', 'Site Title', 'com ex', 1), ('example.com', 'Site Title', 'ex foo', 0), ('example.com', 'Site Title', 'foo com', 0), ('example.com', 'Site Title', 'exm', 0), ('example.com', 'Site Title', 'Si Ti', 1), - ('example.com', 'Site Title', 'Ti Si', 0), + ('example.com', 'Site Title', 'Ti Si', 1), ('example.com', '', 'foo', 0), ('foo_bar', '', '_', 1), ('foobar', '', '_', 0), From a2bcd68d5628bcc1f0df47dd965bedf302855729 Mon Sep 17 00:00:00 2001 From: George Edward Bulmer Date: Mon, 11 Dec 2017 13:35:39 +0000 Subject: [PATCH 147/322] Code review changes. This fixes whitespace and alignment issues, and removes a stray test. --- qutebrowser/browser/commands.py | 4 ++-- qutebrowser/misc/guiprocess.py | 6 +++--- tests/unit/misc/test_guiprocess.py | 21 ++------------------- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/qutebrowser/browser/commands.py b/qutebrowser/browser/commands.py index 495d48414..3075a24da 100644 --- a/qutebrowser/browser/commands.py +++ b/qutebrowser/browser/commands.py @@ -1178,7 +1178,7 @@ class CommandDispatcher: @cmdutils.register(instance='command-dispatcher', scope='window', maxsplit=0, no_replace_variables=True) def spawn(self, cmdline, userscript=False, verbose=False, - output=False, detach=False): + output=False, detach=False): """Spawn a command in a shell. Args: @@ -1218,7 +1218,7 @@ class CommandDispatcher: if output: tabbed_browser = objreg.get('tabbed-browser', scope='window', - window='last-focused') + window='last-focused') tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True) @cmdutils.register(instance='command-dispatcher', scope='window') diff --git a/qutebrowser/misc/guiprocess.py b/qutebrowser/misc/guiprocess.py index 80803f53f..4b74a512d 100644 --- a/qutebrowser/misc/guiprocess.py +++ b/qutebrowser/misc/guiprocess.py @@ -101,7 +101,7 @@ class GUIProcess(QObject): stderr = bytes(self._proc.readAllStandardError()).decode('utf-8') stdout = bytes(self._proc.readAllStandardOutput()).decode('utf-8') - qutescheme.spawn_output = self.spawn_format(code, status, + qutescheme.spawn_output = self._spawn_format(code, status, stdout, stderr) if status == QProcess.CrashExit: @@ -122,7 +122,7 @@ class GUIProcess(QObject): if stderr: log.procs.error("Process stderr:\n" + stderr.strip()) - def spawn_format(self, code=0, status=0, stdout="", stderr=""): + def _spawn_format(self, code=0, status=0, stdout="", stderr=""): """Produce a formatted string for spawn output.""" stdout = (stdout or "(No output)").strip() stderr = (stderr or "(No output)").strip() @@ -130,7 +130,7 @@ class GUIProcess(QObject): spawn_string = ("Process finished with code {}, status {}\n" "\nProcess stdout:\n {}" "\nProcess stderr:\n {}").format(code, status, - stdout, stderr) + stdout, stderr) return spawn_string @pyqtSlot() diff --git a/tests/unit/misc/test_guiprocess.py b/tests/unit/misc/test_guiprocess.py index 69ce3812f..25e46476e 100644 --- a/tests/unit/misc/test_guiprocess.py +++ b/tests/unit/misc/test_guiprocess.py @@ -60,7 +60,7 @@ def test_start(proc, qtbot, message_mock, py_proc): proc.start(*argv) assert not message_mock.messages - assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") + assert qutescheme.spawn_output == proc._spawn_format(stdout="test") def test_start_verbose(proc, qtbot, message_mock, py_proc): @@ -77,24 +77,7 @@ def test_start_verbose(proc, qtbot, message_mock, py_proc): assert msgs[1].level == usertypes.MessageLevel.info assert msgs[0].text.startswith("Executing:") assert msgs[1].text == "Testprocess exited successfully." - assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") - - -def test_start_output(proc, qtbot, message_mock, py_proc): - """Test starting a process verbosely.""" - proc.verbose = True - - with qtbot.waitSignals([proc.started, proc.finished], timeout=10000, - order='strict'): - argv = py_proc("import sys; print('test'); sys.exit(0)") - proc.start(*argv) - - msgs = message_mock.messages - assert msgs[0].level == usertypes.MessageLevel.info - assert msgs[1].level == usertypes.MessageLevel.info - assert msgs[0].text.startswith("Executing:") - assert msgs[1].text == "Testprocess exited successfully." - assert qutescheme.spawn_output == proc.spawn_format(0, 0, stdout="test") + assert qutescheme.spawn_output == proc._spawn_format(stdout="test") def test_start_env(monkeypatch, qtbot, py_proc): From dd7a082265d4e15567c31bf61d1baec0fb232892 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:12 +0100 Subject: [PATCH 148/322] Update codecov from 2.0.9 to 2.0.10 --- misc/requirements/requirements-codecov.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-codecov.txt b/misc/requirements/requirements-codecov.txt index 31c319c39..6601cfb12 100644 --- a/misc/requirements/requirements-codecov.txt +++ b/misc/requirements/requirements-codecov.txt @@ -2,7 +2,7 @@ certifi==2017.11.5 chardet==3.0.4 -codecov==2.0.9 +codecov==2.0.10 coverage==4.4.2 idna==2.6 requests==2.18.4 From 519dc6a7c9f890f2377fc71d716f7d0ddfa4b6ce Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:13 +0100 Subject: [PATCH 149/322] Update setuptools from 38.2.3 to 38.2.4 --- misc/requirements/requirements-pip.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pip.txt b/misc/requirements/requirements-pip.txt index 8ca5a867e..b7914cac5 100644 --- a/misc/requirements/requirements-pip.txt +++ b/misc/requirements/requirements-pip.txt @@ -3,6 +3,6 @@ appdirs==1.4.3 packaging==16.8 pyparsing==2.2.0 -setuptools==38.2.3 +setuptools==38.2.4 six==1.11.0 wheel==0.30.0 From 713a2ef2c169a19db1ab6f7c97e60c2e1d2e2806 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:15 +0100 Subject: [PATCH 150/322] Update pylint from 1.7.4 to 1.7.5 --- misc/requirements/requirements-pylint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-pylint.txt b/misc/requirements/requirements-pylint.txt index cab15c497..346fa4227 100644 --- a/misc/requirements/requirements-pylint.txt +++ b/misc/requirements/requirements-pylint.txt @@ -8,7 +8,7 @@ idna==2.6 isort==4.2.15 lazy-object-proxy==1.3.1 mccabe==0.6.1 -pylint==1.7.4 +pylint==1.7.5 ./scripts/dev/pylint_checkers requests==2.18.4 six==1.11.0 From 5d8e3a969f59c0484bb7f45087166e728b4467d1 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:16 +0100 Subject: [PATCH 151/322] Update cheroot from 5.10.0 to 6.0.0 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 46905f497..62ea92c3c 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -2,7 +2,7 @@ attrs==17.3.0 beautifulsoup4==4.6.0 -cheroot==5.10.0 +cheroot==6.0.0 click==6.7 # colorama==0.3.9 coverage==4.4.2 From 22fe42d38ea76f261aea69e8c8d961517af18b03 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:18 +0100 Subject: [PATCH 152/322] Update hypothesis from 3.40.1 to 3.42.1 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 62ea92c3c..439267282 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -11,7 +11,7 @@ fields==5.0.0 Flask==0.12.2 glob2==0.6 hunter==2.0.2 -hypothesis==3.40.1 +hypothesis==3.42.1 itsdangerous==0.24 # Jinja2==2.10 Mako==1.0.7 From d57a81a3d347975c7cb16a6194261ae320a9b5ad Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:19 +0100 Subject: [PATCH 153/322] Update pytest from 3.3.0 to 3.3.1 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index 439267282..bfb8a5ea3 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -21,7 +21,7 @@ parse-type==0.4.2 pluggy==0.6.0 py==1.5.2 py-cpuinfo==3.3.0 -pytest==3.3.0 +pytest==3.3.1 pytest-bdd==2.19.0 pytest-benchmark==3.1.1 pytest-cov==2.5.1 From d114deac70d50b843291b9d3cfafe122e7afef34 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Mon, 11 Dec 2017 16:23:21 +0100 Subject: [PATCH 154/322] Update werkzeug from 0.12.2 to 0.13 --- misc/requirements/requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/requirements/requirements-tests.txt b/misc/requirements/requirements-tests.txt index bfb8a5ea3..bf80acee7 100644 --- a/misc/requirements/requirements-tests.txt +++ b/misc/requirements/requirements-tests.txt @@ -36,4 +36,4 @@ pytest-xvfb==1.0.0 PyVirtualDisplay==0.2.1 six==1.11.0 vulture==0.26 -Werkzeug==0.12.2 +Werkzeug==0.13 From 481dec067dda7bf6f5a8df783568e7f7ee14923f Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Mon, 11 Dec 2017 17:38:12 +0100 Subject: [PATCH 155/322] Don't override background-color for qutebrowser pages Fixes #3381 --- qutebrowser/html/base.html | 1 - 1 file changed, 1 deletion(-) diff --git a/qutebrowser/html/base.html b/qutebrowser/html/base.html index f21eb227b..6db182908 100644 --- a/qutebrowser/html/base.html +++ b/qutebrowser/html/base.html @@ -10,7 +10,6 @@ vim: ft=html fileencoding=utf-8 sts=4 sw=4 et: '); +//# sourceMappingURL=angular.min.js.map diff --git a/tests/end2end/data/hints/html/angular1.html b/tests/end2end/data/hints/html/angular.html similarity index 84% rename from tests/end2end/data/hints/html/angular1.html rename to tests/end2end/data/hints/html/angular.html index de7712c0b..bc6b10340 100644 --- a/tests/end2end/data/hints/html/angular1.html +++ b/tests/end2end/data/hints/html/angular.html @@ -11,7 +11,7 @@

The button has been clicked {{count}} times.

- +