Merge branch 'master' into new-config

.pyup.yml

@@ -1 +1 @@
-schedule: "every week"
+schedule: "every week on monday"

CONTRIBUTING.asciidoc

@@ -479,7 +479,8 @@ The following arguments are supported for `@cmdutils.argument`:
 - `win_id=True`: Mark the argument as special window ID argument
 - `count=True`: Mark the argument as special count argument
 - `hide=True`: Hide the argument from the documentation
-- `completion`: A `usertypes.Completion` member to use as completion.
+- `completion`: A completion function (see `qutebrowser.completions.models.*`)
+  to use when completing arguments for the given command.
 - `choices`: The allowed string choices for the argument.
 
 The name of an argument will always be the parameter name, with any trailing

INSTALL.asciidoc

@@ -85,7 +85,7 @@ qutebrowser is available in the official repositories for Fedora 22 and newer.
 # dnf install qutebrowser
 ----
 
-It's also recommended to install `qt5-qtwebengine` and start with `--backend
+It's also recommended to install `python3-qt5-webengine` and start with `--backend
 webengine` to use the new backend.
 
 On Archlinux
@@ -175,6 +175,12 @@ If video or sound don't seem to work, try installing the gstreamer plugins:
 # emerge -av gst-plugins-{base,good,bad,ugly,libav}
 ----
 
+To be able to play videos with proprietary codecs with QtWebEngine, you will
+need to turn off the `bindist` flag for `dev-qt/qtwebengine`.
+
+See the https://wiki.gentoo.org/wiki/Qutebrowser#USE_flags[Gentoo Wiki] for
+more information.
+
 
 On Void Linux
 -------------

misc/requirements/requirements-codecov.txt

@@ -5,5 +5,5 @@ chardet==3.0.4
 codecov==2.0.9
 coverage==4.4.1
 idna==2.6
-requests==2.18.3
+requests==2.18.4
 urllib3==1.22

misc/requirements/requirements-pylint-master.txt

@@ -10,7 +10,7 @@ lazy-object-proxy==1.3.1
 mccabe==0.6.1
 -e git+https://github.com/PyCQA/pylint.git#egg=pylint
 ./scripts/dev/pylint_checkers
-requests==2.18.3
+requests==2.18.4
 six==1.10.0
 uritemplate==3.0.0
 uritemplate.py==3.0.2

misc/requirements/requirements-pylint.txt

@@ -10,7 +10,7 @@ lazy-object-proxy==1.3.1
 mccabe==0.6.1
 pylint==1.7.2
 ./scripts/dev/pylint_checkers
-requests==2.18.3
+requests==2.18.4
 six==1.10.0
 uritemplate==3.0.0
 uritemplate.py==3.0.2

misc/requirements/requirements-tests.txt

@@ -9,10 +9,10 @@ decorator==4.1.2
 EasyProcess==0.2.3
 fields==5.0.0
 Flask==0.12.2
-glob2==0.5
+glob2==0.6
 httpbin==0.5.0
 hunter==1.4.1
-hypothesis==3.18.0
+hypothesis==3.19.1
 itsdangerous==0.24
 # Jinja2==2.9.6
 Mako==1.0.7
@@ -30,10 +30,10 @@ pytest-instafail==0.3.0
 pytest-mock==1.6.2
 pytest-qt==2.1.2
 pytest-repeat==0.4.1
-pytest-rerunfailures==2.2
+pytest-rerunfailures==3.0
 pytest-travis-fold==1.2.0
 pytest-xvfb==1.0.0
 PyVirtualDisplay==0.2.1
 six==1.10.0
-vulture==0.24
+vulture==0.25
 Werkzeug==0.12.2

misc/requirements/requirements-vulture.txt

@@ -1,3 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-vulture==0.24
+vulture==0.25

misc/userscripts/format_json

@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Behavior:
+#   Userscript for qutebrowser which will take the raw JSON text of the current
+#   page, format it using `jq`, will add syntax highlighting using `pygments`,
+#   and open the syntax highlighted pretty printed html in a new tab. If the file
+#   is larger than 10MB then this script will only indent the json and will forego
+#   syntax highlighting using pygments.
+#
+#   In order to use this script, just start it using `spawn --userscript` from
+#   qutebrowser. I recommend using an alias, e.g. put this in the
+#   [alias]-section of qutebrowser.conf:
+#
+#     json = spawn --userscript /path/to/json_format
+#
+#   Note that the color style defaults to monokai, but a different pygments style
+#   can be passed as the first parameter to the script. A full list of the pygments
+#   styles can be found at: https://help.farbox.com/pygments.html
+#
+# Bryan Gilbert, 2017
+
+# default style to monokai if none is provided
+STYLE=${1:-monokai}
+# format json using jq
+FORMATTED_JSON="$(cat "$QUTE_TEXT" | jq '.')"
+
+# if jq command failed or formatted json is empty, assume failure and terminate
+if [ $? -ne 0 ] || [ -z "$FORMATTED_JSON" ]; then
+    echo "Invalid json, aborting..."
+    exit 1
+fi
+
+# calculate the filesize of the json document
+FILE_SIZE=$(ls -s --block-size=1048576 "$QUTE_TEXT" | cut -d' ' -f1)
+
+# use pygments to pretty-up the json (syntax highlight) if file is less than 10MB
+if [ "$FILE_SIZE" -lt "10" ]; then
+    FORMATTED_JSON="$(echo "$FORMATTED_JSON" | pygmentize -l json -f html -O full,style=$STYLE)"
+fi
+
+# create a temp file and write the formatted json to that file
+TEMP_FILE="$(mktemp --suffix '.html')"
+echo "$FORMATTED_JSON" > $TEMP_FILE
+
+
+# send the command to qutebrowser to open the new file containing the formatted json
+echo "open -t file://$TEMP_FILE" >> "$QUTE_FIFO"

misc/userscripts/password_fill

@@ -327,6 +327,17 @@ open_entry "$file"
 
 js() {
 cat <<EOF
+    function isVisible(elem) {
+        var style = elem.ownerDocument.defaultView.getComputedStyle(elem, null);
+
+        if (style.getPropertyValue("visibility") !== "visible" ||
+            style.getPropertyValue("display") === "none" ||
+            style.getPropertyValue("opacity") === "0") {
+            return false;
+        }
+
+        return elem.offsetWidth > 0 && elem.offsetHeight > 0;
+    };
     function hasPasswordField(form) {
         var inputs = form.getElementsByTagName("input");
         for (var j = 0; j < inputs.length; j++) {
@@ -341,7 +352,7 @@ cat <<EOF
         var inputs = form.getElementsByTagName("input");
         for (var j = 0; j < inputs.length; j++) {
             var input = inputs[j];
-            if (input.type == "text" || input.type == "email") {
+            if (isVisible(input) && (input.type == "text" || input.type == "email")) {
                 input.focus();
                 input.value = "$(javascript_escape "${username}")";
                 input.blur();

qutebrowser/browser/history.py

@@ -30,6 +30,10 @@ from qutebrowser.utils import (utils, objreg, log, usertypes, message,
 from qutebrowser.misc import objects, sql
 
 
+# increment to indicate that HistoryCompletion must be regenerated
+_USER_VERSION = 1
+
+
 class CompletionHistory(sql.SqlTable):
 
     """History which only has the newest entry for each URL."""
@@ -48,6 +52,11 @@ class WebHistory(sql.SqlTable):
         super().__init__("History", ['url', 'title', 'atime', 'redirect'],
                          parent=parent)
         self.completion = CompletionHistory(parent=self)
+        if sql.Query('pragma user_version').run().value() < _USER_VERSION:
+            self.completion.delete_all()
+        if not self.completion:
+            # either the table is out-of-date or the user wiped it manually
+            self._rebuild_completion()
         self.create_index('HistoryIndex', 'url')
         self.create_index('HistoryAtimeIndex', 'atime')
         self._contains_query = self.contains_query('url')
@@ -71,6 +80,18 @@ class WebHistory(sql.SqlTable):
     def __contains__(self, url):
         return self._contains_query.run(val=url).value()
 
+    def _rebuild_completion(self):
+        data = {'url': [], 'title': [], 'last_atime': []}
+        # select the latest entry for each url
+        q = sql.Query('SELECT url, title, max(atime) AS atime FROM History '
+                      'WHERE NOT redirect GROUP BY url ORDER BY atime asc')
+        for entry in q.run():
+            data['url'].append(self._format_completion_url(QUrl(entry.url)))
+            data['title'].append(entry.title)
+            data['last_atime'].append(entry.atime)
+        self.completion.insert_batch(data, replace=True)
+        sql.Query('pragma user_version = {}'.format(_USER_VERSION)).run()
+
     def get_recent(self):
         """Get the most recent history entries."""
         return self.select(sort_by='atime', sort_order='desc', limit=100)
@@ -159,13 +180,12 @@ class WebHistory(sql.SqlTable):
             return
 
         atime = int(atime) if (atime is not None) else int(time.time())
-        url_str = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)
-        self.insert({'url': url_str,
+        self.insert({'url': self._format_url(url),
                      'title': title,
                      'atime': atime,
                      'redirect': redirect})
         if not redirect:
-            self.completion.insert({'url': url_str,
+            self.completion.insert({'url': self._format_completion_url(url),
                                     'title': title,
                                     'last_atime': atime},
                                    replace=True)
@@ -249,12 +269,13 @@ class WebHistory(sql.SqlTable):
                     if parsed is None:
                         continue
                     url, title, atime, redirect = parsed
-                    data['url'].append(url)
+                    data['url'].append(self._format_url(url))
                     data['title'].append(title)
                     data['atime'].append(atime)
                     data['redirect'].append(redirect)
                     if not redirect:
-                        completion_data['url'].append(url)
+                        completion_data['url'].append(
+                            self._format_completion_url(url))
                         completion_data['title'].append(title)
                         completion_data['last_atime'].append(atime)
                 except ValueError as ex:
@@ -263,6 +284,12 @@ class WebHistory(sql.SqlTable):
         self.insert_batch(data)
         self.completion.insert_batch(completion_data, replace=True)
 
+    def _format_url(self, url):
+        return url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
+
+    def _format_completion_url(self, url):
+        return url.toString(QUrl.RemovePassword)
+
     @cmdutils.register(instance='web-history', debug=True)
     def debug_dump_history(self, dest):
         """Dump the history to a file in the old pre-SQL format.

qutebrowser/javascript/.eslintrc.yaml

@@ -43,3 +43,4 @@ rules:
      array-bracket-newline: "off"
      array-element-newline: "off"
      no-multi-spaces: ["error", {"ignoreEOLComments": true}]
+     function-paren-newline: "off"

qutebrowser/misc/sessions.py

@@ -480,7 +480,7 @@ class SessionManager(QObject):
         try:
             if only_active_window:
                 name = self.save(name, only_window=win_id,
-                                 with_private=with_private)
+                                 with_private=True)
             else:
                 name = self.save(name, with_private=with_private)
         except SessionError as e:

scripts/dev/run_vulture.py

@@ -132,10 +132,11 @@ def report(items):
     properties which get used for the items.
     """
     output = []
-    for item in sorted(items, key=lambda e: (e.filename.lower(), e.lineno)):
+    for item in sorted(items,
+                       key=lambda e: (e.filename.lower(), e.first_lineno)):
         relpath = os.path.relpath(item.filename)
         path = relpath if not relpath.startswith('..') else item.filename
-        output.append("{}:{}: Unused {} '{}'".format(path, item.lineno,
+        output.append("{}:{}: Unused {} '{}'".format(path, item.first_lineno,
                                                      item.typ, item.name))
     return output
 
@@ -148,7 +149,7 @@ def run(files):
 
         whitelist_file.close()
 
-        vult = vulture.Vulture(exclude=[], verbose=False)
+        vult = vulture.Vulture(verbose=False)
         vult.scavenge(files + [whitelist_file.name])
 
         os.remove(whitelist_file.name)

tests/end2end/features/private.feature

@@ -153,3 +153,26 @@ Feature: Using private browsing
               - history:
                 - url: http://localhost:*/data/numbers/1.txt
                 - url: http://localhost:*/data/numbers/2.txt
+
+
+  Scenario: Saving a private session with only-active-window
+        When I open data/numbers/1.txt
+        And I open data/numbers/2.txt in a new tab
+        And I open data/numbers/3.txt in a private window
+        And I open data/numbers/4.txt in a new tab
+        And I open data/numbers/5.txt in a new tab
+        And I run :session-save --only-active-window window_session_name
+        And I run :window-only
+        And I run :tab-only
+        And I run :session-load -c window_session_name
+        And I wait until data/numbers/5.txt is loaded
+        Then the session should look like:
+            windows:
+                - tabs:
+                    - history:
+                        - url: http://localhost:*/data/numbers/3.txt
+                    - history:
+                        - url: http://localhost:*/data/numbers/4.txt
+                    - history:
+                        - active: true
+                          url: http://localhost:*/data/numbers/5.txt

tests/unit/browser/test_history.py

@@ -143,23 +143,27 @@ def test_delete_url(hist):
     assert completion_diff == {('http://example.com/1', '', 0)}
 
 
-@pytest.mark.parametrize('url, atime, title, redirect, expected_url', [
-    ('http://www.example.com', 12346, 'the title', False,
-        'http://www.example.com'),
-    ('http://www.example.com', 12346, 'the title', True,
-        'http://www.example.com'),
-    ('http://www.example.com/spa ce', 12346, 'the title', False,
-        'http://www.example.com/spa%20ce'),
-    ('https://user:pass@example.com', 12346, 'the title', False,
-        'https://user@example.com'),
-])
-def test_add_url(qtbot, hist, url, atime, title, redirect, expected_url):
+@pytest.mark.parametrize(
+    'url, atime, title, redirect, history_url, completion_url', [
+
+        ('http://www.example.com', 12346, 'the title', False,
+            'http://www.example.com', 'http://www.example.com'),
+        ('http://www.example.com', 12346, 'the title', True,
+            'http://www.example.com', None),
+        ('http://www.example.com/sp ce', 12346, 'the title', False,
+            'http://www.example.com/sp%20ce', 'http://www.example.com/sp ce'),
+        ('https://user:pass@example.com', 12346, 'the title', False,
+            'https://user@example.com', 'https://user@example.com'),
+    ]
+)
+def test_add_url(qtbot, hist, url, atime, title, redirect, history_url,
+                 completion_url):
     hist.add_url(QUrl(url), atime=atime, title=title, redirect=redirect)
-    assert list(hist) == [(expected_url, title, atime, redirect)]
-    if redirect:
+    assert list(hist) == [(history_url, title, atime, redirect)]
+    if completion_url is None:
         assert not len(hist.completion)
     else:
-        assert list(hist.completion) == [(expected_url, title, atime)]
+        assert list(hist.completion) == [(completion_url, title, atime)]
 
 
 def test_add_url_invalid(qtbot, hist, caplog):
@@ -349,3 +353,50 @@ def test_debug_dump_history_nonexistent(hist, tmpdir):
     histfile = tmpdir / 'nonexistent' / 'history'
     with pytest.raises(cmdexc.CommandError):
         hist.debug_dump_history(str(histfile))
+
+
+def test_rebuild_completion(hist):
+    hist.insert({'url': 'example.com/1', 'title': 'example1',
+                 'redirect': False, 'atime': 1})
+    hist.insert({'url': 'example.com/1', 'title': 'example1',
+                 'redirect': False, 'atime': 2})
+    hist.insert({'url': 'example.com/2%203', 'title': 'example2',
+                 'redirect': False, 'atime': 3})
+    hist.insert({'url': 'example.com/3', 'title': 'example3',
+                 'redirect': True, 'atime': 4})
+    hist.insert({'url': 'example.com/2 3', 'title': 'example2',
+                 'redirect': False, 'atime': 5})
+    hist.completion.delete_all()
+
+    hist2 = history.WebHistory()
+    assert list(hist2.completion) == [
+        ('example.com/1', 'example1', 2),
+        ('example.com/2 3', 'example2', 5),
+    ]
+
+
+def test_no_rebuild_completion(hist):
+    """Ensure that completion is not regenerated unless completely empty."""
+    hist.add_url(QUrl('example.com/1'), redirect=False, atime=1)
+    hist.add_url(QUrl('example.com/2'), redirect=False, atime=2)
+    hist.completion.delete('url', 'example.com/2')
+
+    hist2 = history.WebHistory()
+    assert list(hist2.completion) == [('example.com/1', '', 1)]
+
+
+def test_user_version(hist, monkeypatch):
+    """Ensure that completion is regenerated if user_version is incremented."""
+    hist.add_url(QUrl('example.com/1'), redirect=False, atime=1)
+    hist.add_url(QUrl('example.com/2'), redirect=False, atime=2)
+    hist.completion.delete('url', 'example.com/2')
+
+    hist2 = history.WebHistory()
+    assert list(hist2.completion) == [('example.com/1', '', 1)]
+
+    monkeypatch.setattr(history, '_USER_VERSION', history._USER_VERSION + 1)
+    hist3 = history.WebHistory()
+    assert list(hist3.completion) == [
+        ('example.com/1', '', 1),
+        ('example.com/2', '', 2),
+    ]