From 790b34dc56342cfd21f4bd28736db1b2ca3f3a79 Mon Sep 17 00:00:00 2001 From: Jesko Date: Tue, 7 Aug 2018 21:59:15 +0200 Subject: [PATCH 1/9] Adding match pattern test esp. IPv6 --- tests/unit/utils/test_urlmatch.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/test_urlmatch.py b/tests/unit/utils/test_urlmatch.py index dcd703790..071a5fbe6 100644 --- a/tests/unit/utils/test_urlmatch.py +++ b/tests/unit/utils/test_urlmatch.py @@ -154,10 +154,14 @@ class TestMatchAllPagesForGivenScheme: @pytest.mark.parametrize('url, expected', [ ("http://google.com", True), + ("http://google.com:80", True), + ("http://google.com.", True), ("http://yahoo.com", True), ("http://google.com/foo", True), ("https://google.com", False), ("http://74.125.127.100/search", True), + ("http://[fc2e:0e35:bb88::edac]", True), + ("http://[::1]/bar", True), ]) def test_urls(self, up, url, expected): assert up.matches(QUrl(url)) == expected @@ -238,6 +242,7 @@ class TestMatchIpAddresses: @pytest.mark.parametrize('pattern, host, match_subdomains', [ ("http://127.0.0.1/*", "127.0.0.1", False), ("http://*.0.0.1/*", "0.0.1", True), + ("http://[::1]/*", "::1", False), ]) def test_attrs(self, pattern, host, match_subdomains): up = urlmatch.UrlPattern(pattern) @@ -255,7 +260,7 @@ class TestMatchIpAddresses: def test_urls(self, pattern, expected): up = urlmatch.UrlPattern(pattern) assert up.matches(QUrl("http://127.0.0.1")) == expected - + class TestMatchChromeUrls: From 599ac5ca233743caf65c8e898fc81aff318c407a Mon Sep 17 00:00:00 2001 From: Jesko Date: Tue, 7 Aug 2018 22:35:00 +0200 Subject: [PATCH 2/9] Adding parser tests for IPv4 --- tests/unit/utils/test_urlmatch.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/test_urlmatch.py b/tests/unit/utils/test_urlmatch.py index 071a5fbe6..16ab0acc9 100644 --- a/tests/unit/utils/test_urlmatch.py +++ b/tests/unit/utils/test_urlmatch.py @@ -84,6 +84,18 @@ from qutebrowser.utils import urlmatch # Additional tests ("http://[", "Invalid IPv6 URL"), + ("http://[fc2e::bb88::edac]:", "Invalid port: Port is empty"), + ("http://[fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac]", "Invalid IPv6 URL"), + ("http://[fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac]", "Invalid IPv6 URL"), + ("http://[127.0.0.1:fc2e::bb88:edac]", "Invalid IPv6 URL"), + ("http://[]:20", "Pattern without host"), + ("http://[fc2e::bb88", "Invalid IPv6 URL"), + ("http://[[fc2e::bb88:edac]", "Invalid IPv6 URL"), + ("http://[fc2e::bb88:edac]]", "Invalid IPv6 URL"), + ("http://[fc2e:bb88:edac]", "Invalid IPv6 URL"), + ("http://[fc2e:bb88:edac::z]", "Invalid IPv6 URL"), + ("http://[fc2e:bb88:edac::2]:2a2", "Invalid port: invalid literal for int() with base 10: '2a2'"), + ]) def test_invalid_patterns(pattern, error): with pytest.raises(urlmatch.ParseError, match=re.escape(error)): @@ -161,6 +173,8 @@ class TestMatchAllPagesForGivenScheme: ("https://google.com", False), ("http://74.125.127.100/search", True), ("http://[fc2e:0e35:bb88::edac]", True), + ("http://[fc2e:e35:bb88::edac]", True), + ("http://[fc2e:e35:bb88::127.0.0.1]", True), ("http://[::1]/bar", True), ]) def test_urls(self, up, url, expected): @@ -260,7 +274,7 @@ class TestMatchIpAddresses: def test_urls(self, pattern, expected): up = urlmatch.UrlPattern(pattern) assert up.matches(QUrl("http://127.0.0.1")) == expected - + class TestMatchChromeUrls: From 4ce5d99b24313bc2fc635f1f9c9d4aaf3ea4b8a7 Mon Sep 17 00:00:00 2001 From: Jesko Date: Wed, 8 Aug 2018 16:55:43 +0200 Subject: [PATCH 3/9] match minimizes ipv6 urls --- qutebrowser/utils/urlmatch.py | 8 ++++++++ tests/unit/utils/test_urlmatch.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/qutebrowser/utils/urlmatch.py b/qutebrowser/utils/urlmatch.py index 5d9afc13e..f8e06fb06 100644 --- a/qutebrowser/utils/urlmatch.py +++ b/qutebrowser/utils/urlmatch.py @@ -30,6 +30,7 @@ import fnmatch import urllib.parse from qutebrowser.utils import utils, qtutils +from PyQt5.QtCore import QUrl class ParseError(Exception): @@ -177,6 +178,13 @@ class UrlPattern: assert self._host is None return + if not utils.raises(ValueError, ipaddress.IPv6Address, parsed.netloc[1:-1]): + # Using QUrl parsing to minimize ipv6 addresses + url = QUrl() + url.setHost(parsed.hostname) + self._host = url.host() + return + # FIXME what about multiple dots? host_parts = parsed.hostname.rstrip('.').split('.') if host_parts[0] == '*': diff --git a/tests/unit/utils/test_urlmatch.py b/tests/unit/utils/test_urlmatch.py index 16ab0acc9..1dadc2cab 100644 --- a/tests/unit/utils/test_urlmatch.py +++ b/tests/unit/utils/test_urlmatch.py @@ -257,6 +257,9 @@ class TestMatchIpAddresses: ("http://127.0.0.1/*", "127.0.0.1", False), ("http://*.0.0.1/*", "0.0.1", True), ("http://[::1]/*", "::1", False), + ("http://[0::1]/*", "::1", False), + ("http://[::01]/*", "::1", False), + ("http://[0:0:0:0:20::1]/*", "::20:0:0:1", False), ]) def test_attrs(self, pattern, host, match_subdomains): up = urlmatch.UrlPattern(pattern) From 484a22f1119330367b194c086458b4f17dded244 Mon Sep 17 00:00:00 2001 From: Jesko Date: Wed, 8 Aug 2018 17:53:14 +0200 Subject: [PATCH 4/9] now declining more wrong ipv6 addresses and cleaning up code --- qutebrowser/utils/urlmatch.py | 9 ++++++--- tests/unit/utils/test_urlmatch.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/qutebrowser/utils/urlmatch.py b/qutebrowser/utils/urlmatch.py index f8e06fb06..4b6b9aa83 100644 --- a/qutebrowser/utils/urlmatch.py +++ b/qutebrowser/utils/urlmatch.py @@ -29,9 +29,10 @@ import ipaddress import fnmatch import urllib.parse -from qutebrowser.utils import utils, qtutils from PyQt5.QtCore import QUrl +from qutebrowser.utils import utils, qtutils + class ParseError(Exception): @@ -178,10 +179,12 @@ class UrlPattern: assert self._host is None return - if not utils.raises(ValueError, ipaddress.IPv6Address, parsed.netloc[1:-1]): + if parsed.netloc.startswith('['): # Using QUrl parsing to minimize ipv6 addresses url = QUrl() - url.setHost(parsed.hostname) + url.setHost("[" + parsed.hostname + "]") + if url.host() == "": + raise ParseError("Invalid IPv6 URL"+parsed.hostname) self._host = url.host() return diff --git a/tests/unit/utils/test_urlmatch.py b/tests/unit/utils/test_urlmatch.py index 1dadc2cab..b8ab4ace1 100644 --- a/tests/unit/utils/test_urlmatch.py +++ b/tests/unit/utils/test_urlmatch.py @@ -84,7 +84,8 @@ from qutebrowser.utils import urlmatch # Additional tests ("http://[", "Invalid IPv6 URL"), - ("http://[fc2e::bb88::edac]:", "Invalid port: Port is empty"), + ("http://[fc2e:bb88::edac]:", "Invalid port: Port is empty"), + ("http://[fc2e::bb88::edac]", "Invalid IPv6 URL"), ("http://[fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac]", "Invalid IPv6 URL"), ("http://[fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac]", "Invalid IPv6 URL"), ("http://[127.0.0.1:fc2e::bb88:edac]", "Invalid IPv6 URL"), From 972b3ae960accff04888bfdcd079d250fd7a3e29 Mon Sep 17 00:00:00 2001 From: Jesko Date: Wed, 8 Aug 2018 18:30:12 +0200 Subject: [PATCH 5/9] changing suggested flaws --- qutebrowser/utils/urlmatch.py | 4 ++-- tests/unit/utils/test_urlmatch.py | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/qutebrowser/utils/urlmatch.py b/qutebrowser/utils/urlmatch.py index 4b6b9aa83..49f6f80f1 100644 --- a/qutebrowser/utils/urlmatch.py +++ b/qutebrowser/utils/urlmatch.py @@ -183,8 +183,8 @@ class UrlPattern: # Using QUrl parsing to minimize ipv6 addresses url = QUrl() url.setHost("[" + parsed.hostname + "]") - if url.host() == "": - raise ParseError("Invalid IPv6 URL"+parsed.hostname) + if not url.isValid(): + raise ParseError(url.errorString()) self._host = url.host() return diff --git a/tests/unit/utils/test_urlmatch.py b/tests/unit/utils/test_urlmatch.py index b8ab4ace1..ea238303f 100644 --- a/tests/unit/utils/test_urlmatch.py +++ b/tests/unit/utils/test_urlmatch.py @@ -85,16 +85,17 @@ from qutebrowser.utils import urlmatch # Additional tests ("http://[", "Invalid IPv6 URL"), ("http://[fc2e:bb88::edac]:", "Invalid port: Port is empty"), - ("http://[fc2e::bb88::edac]", "Invalid IPv6 URL"), - ("http://[fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac]", "Invalid IPv6 URL"), - ("http://[fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac]", "Invalid IPv6 URL"), - ("http://[127.0.0.1:fc2e::bb88:edac]", "Invalid IPv6 URL"), + ("http://[fc2e::bb88::edac]", """Invalid IPv6 address (character ':' not permitted); source was "[fc2e::bb88::edac]"; host = """""), + ("http://[fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac]", 'Invalid IPv6 address; source was "[fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac]"; host = ""'), + ("http://[fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac]", 'Invalid IPv6 address; source was "[fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac]"; host = ""'), + ("http://[127.0.0.1:fc2e::bb88:edac]", """Invalid IPv6 address (character '1' not permitted); source was "[127.0.0.1:fc2e::bb88:edac]"; host = """""), ("http://[]:20", "Pattern without host"), ("http://[fc2e::bb88", "Invalid IPv6 URL"), - ("http://[[fc2e::bb88:edac]", "Invalid IPv6 URL"), - ("http://[fc2e::bb88:edac]]", "Invalid IPv6 URL"), - ("http://[fc2e:bb88:edac]", "Invalid IPv6 URL"), - ("http://[fc2e:bb88:edac::z]", "Invalid IPv6 URL"), + ("http://[[fc2e::bb88:edac]", """Invalid IPv6 address (character '[' not permitted); source was "[[fc2e::bb88:edac]"; host = """""), + pytest.param("http://[fc2e::bb88:edac]]", "Invalid IPv6 URL", marks=pytest.mark.xfail( + reason="https://bugs.python.org/issue34360")), + ("http://[fc2e:bb88:edac]", 'Invalid IPv6 address; source was "[fc2e:bb88:edac]"; host = ""'), + ("http://[fc2e:bb88:edac::z]", """Invalid IPv6 address (character 'z' not permitted); source was "[fc2e:bb88:edac::z]"; host = """""), ("http://[fc2e:bb88:edac::2]:2a2", "Invalid port: invalid literal for int() with base 10: '2a2'"), ]) From 0b514373466eec79bb3ff0ef08ffc0ecd9d85849 Mon Sep 17 00:00:00 2001 From: Jesko Date: Fri, 10 Aug 2018 19:05:15 +0200 Subject: [PATCH 6/9] fixing indentation --- tests/unit/utils/test_urlmatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/utils/test_urlmatch.py b/tests/unit/utils/test_urlmatch.py index ea238303f..af1e9eed8 100644 --- a/tests/unit/utils/test_urlmatch.py +++ b/tests/unit/utils/test_urlmatch.py @@ -92,7 +92,7 @@ from qutebrowser.utils import urlmatch ("http://[]:20", "Pattern without host"), ("http://[fc2e::bb88", "Invalid IPv6 URL"), ("http://[[fc2e::bb88:edac]", """Invalid IPv6 address (character '[' not permitted); source was "[[fc2e::bb88:edac]"; host = """""), - pytest.param("http://[fc2e::bb88:edac]]", "Invalid IPv6 URL", marks=pytest.mark.xfail( + pytest.param("http://[fc2e::bb88:edac]]", "Invalid IPv6 URL", marks=pytest.mark.xfail( reason="https://bugs.python.org/issue34360")), ("http://[fc2e:bb88:edac]", 'Invalid IPv6 address; source was "[fc2e:bb88:edac]"; host = ""'), ("http://[fc2e:bb88:edac::z]", """Invalid IPv6 address (character 'z' not permitted); source was "[fc2e:bb88:edac::z]"; host = """""), From b74b069153e5d54acec2659368a1f5e69c164ace Mon Sep 17 00:00:00 2001 From: Jesko Date: Fri, 10 Aug 2018 19:14:48 +0200 Subject: [PATCH 7/9] removing useless brackets --- qutebrowser/utils/urlmatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qutebrowser/utils/urlmatch.py b/qutebrowser/utils/urlmatch.py index 49f6f80f1..02db96e33 100644 --- a/qutebrowser/utils/urlmatch.py +++ b/qutebrowser/utils/urlmatch.py @@ -182,7 +182,7 @@ class UrlPattern: if parsed.netloc.startswith('['): # Using QUrl parsing to minimize ipv6 addresses url = QUrl() - url.setHost("[" + parsed.hostname + "]") + url.setHost(parsed.hostname) if not url.isValid(): raise ParseError(url.errorString()) self._host = url.host() From a676cca6c5aeae5810c4d3f8b360a7e0dfd5ee39 Mon Sep 17 00:00:00 2001 From: Jesko Date: Fri, 10 Aug 2018 20:49:20 +0200 Subject: [PATCH 8/9] changing the texts of the errors --- qutebrowser/config/configdata.py | 2 +- qutebrowser/config/configdata.yml | 8 ++++++++ tests/unit/utils/test_urlmatch.py | 17 ++++++++--------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/qutebrowser/config/configdata.py b/qutebrowser/config/configdata.py index da3025319..dace0772a 100644 --- a/qutebrowser/config/configdata.py +++ b/qutebrowser/config/configdata.py @@ -56,7 +56,7 @@ class Option: @attr.s class Migrations: - """Nigrated options in configdata.yml. + """Migrated options in configdata.yml. Attributes: renamed: A dict mapping old option names to new names. diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index e57459f64..6131be6ba 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1622,6 +1622,14 @@ url.yank_ignored_parameters: - utm_content desc: URL parameters to strip with `:yank url`. +url.yank_remove_password: + type: Bool + default: true + desc: | + Remove the password with `:yank url`. + + E.g. + ## window window.hide_wayland_decoration: diff --git a/tests/unit/utils/test_urlmatch.py b/tests/unit/utils/test_urlmatch.py index af1e9eed8..b77588469 100644 --- a/tests/unit/utils/test_urlmatch.py +++ b/tests/unit/utils/test_urlmatch.py @@ -85,17 +85,16 @@ from qutebrowser.utils import urlmatch # Additional tests ("http://[", "Invalid IPv6 URL"), ("http://[fc2e:bb88::edac]:", "Invalid port: Port is empty"), - ("http://[fc2e::bb88::edac]", """Invalid IPv6 address (character ':' not permitted); source was "[fc2e::bb88::edac]"; host = """""), - ("http://[fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac]", 'Invalid IPv6 address; source was "[fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac]"; host = ""'), - ("http://[fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac]", 'Invalid IPv6 address; source was "[fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac]"; host = ""'), - ("http://[127.0.0.1:fc2e::bb88:edac]", """Invalid IPv6 address (character '1' not permitted); source was "[127.0.0.1:fc2e::bb88:edac]"; host = """""), + ("http://[fc2e::bb88::edac]", 'Invalid IPv6 address; source was "fc2e::bb88::edac"; host = ""'), + ("http://[fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac]", 'Invalid IPv6 address; source was "fc2e:0e35:bb88::edac:fc2e:0e35:bb88:edac"; host = ""'), + ("http://[fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac]", 'Invalid IPv6 address; source was "fc2e:0e35:bb88:af:edac:fc2e:0e35:bb88:edac"; host = ""'), + ("http://[127.0.0.1:fc2e::bb88:edac]", 'Invalid IPv6 address; source was "127.0.0.1:fc2e::bb88:edac'), ("http://[]:20", "Pattern without host"), ("http://[fc2e::bb88", "Invalid IPv6 URL"), - ("http://[[fc2e::bb88:edac]", """Invalid IPv6 address (character '[' not permitted); source was "[[fc2e::bb88:edac]"; host = """""), - pytest.param("http://[fc2e::bb88:edac]]", "Invalid IPv6 URL", marks=pytest.mark.xfail( - reason="https://bugs.python.org/issue34360")), - ("http://[fc2e:bb88:edac]", 'Invalid IPv6 address; source was "[fc2e:bb88:edac]"; host = ""'), - ("http://[fc2e:bb88:edac::z]", """Invalid IPv6 address (character 'z' not permitted); source was "[fc2e:bb88:edac::z]"; host = """""), + ("http://[[fc2e::bb88:edac]", """Expected ']' to match '[' in hostname; source was "[fc2e::bb88:edac"; host = """""), + pytest.param("http://[fc2e::bb88:edac]]", "Invalid IPv6 URL", marks=pytest.mark.xfail(reason="https://bugs.python.org/issue34360")), + ("http://[fc2e:bb88:edac]", 'Invalid IPv6 address; source was "fc2e:bb88:edac"; host = ""'), + ("http://[fc2e:bb88:edac::z]", 'Invalid IPv6 address; source was "fc2e:bb88:edac::z"; host = ""'), ("http://[fc2e:bb88:edac::2]:2a2", "Invalid port: invalid literal for int() with base 10: '2a2'"), ]) From 0d785334808baa40a2849912470cbb8d3513ab83 Mon Sep 17 00:00:00 2001 From: Jesko Date: Sat, 11 Aug 2018 19:43:11 +0200 Subject: [PATCH 9/9] remove useless config option --- qutebrowser/config/configdata.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/qutebrowser/config/configdata.yml b/qutebrowser/config/configdata.yml index 6131be6ba..e57459f64 100644 --- a/qutebrowser/config/configdata.yml +++ b/qutebrowser/config/configdata.yml @@ -1622,14 +1622,6 @@ url.yank_ignored_parameters: - utm_content desc: URL parameters to strip with `:yank url`. -url.yank_remove_password: - type: Bool - default: true - desc: | - Remove the password with `:yank url`. - - E.g. - ## window window.hide_wayland_decoration: