diff --git a/qutebrowser/utils/urlmatch.py b/qutebrowser/utils/urlmatch.py
index 4bf02a3b8..faae3b091 100644
--- a/qutebrowser/utils/urlmatch.py
+++ b/qutebrowser/utils/urlmatch.py
@@ -87,10 +87,8 @@ class UrlPattern:
             parsed = urllib.parse.urlparse(pattern)
         except ValueError as e:
             raise ParseError(str(e))
-        # "Changed in version 3.6: Out-of-range port numbers now raise
-        # ValueError, instead of returning None."
-        if parsed is None:
-            raise ParseError("Failed to parse {}".format(pattern))
+
+        assert parsed is not None
 
         self._init_scheme(parsed)
         self._init_host(parsed)
@@ -166,12 +164,12 @@ class UrlPattern:
             # We can't access parsed.port as it tries to run int()
             self._port = None
         elif parsed.netloc.endswith(':'):
-            raise ParseError("Empty port")
+            raise ParseError("Invalid port: Port is empty")
         else:
             try:
                 self._port = parsed.port
-            except ValueError:
-                raise ParseError("Invalid port")
+            except ValueError as e:
+                raise ParseError("Invalid port: {}".format(e))
 
         if (self._scheme not in list(self.DEFAULT_PORTS) + [None] and
                 self._port is not None):
diff --git a/tests/unit/utils/test_urlmatch.py b/tests/unit/utils/test_urlmatch.py
index e60885516..67368933d 100644
--- a/tests/unit/utils/test_urlmatch.py
+++ b/tests/unit/utils/test_urlmatch.py
@@ -29,6 +29,8 @@ Currently not tested:
 - Any other features we don't need, such as .GetAsString() or set operations.
 """
 
+import re
+import sys
 import string
 
 import pytest
@@ -66,18 +68,24 @@ from qutebrowser.utils import urlmatch
     ("http://foo.*/bar", "TLD wildcards are not implemented yet"),
 
     # Chromium: PARSE_ERROR_INVALID_PORT
-    ("http://foo:/", "Empty port"),
-    ("http://*.foo:/", "Empty port"),
-    ("http://foo:com/", "Invalid port"),
-    ("http://foo:123456/", "Invalid port"),
-    ("http://foo:80:80/monkey", "Invalid port"),
+    ("http://foo:/", "Invalid port: Port is empty"),
+    ("http://*.foo:/", "Invalid port: Port is empty"),
+    ("http://foo:com/",
+     "Invalid port: invalid literal for int() with base 10: 'com'"),
+    pytest.param("http://foo:123456/",
+                 "Invalid port: Port out of range 0-65535",
+                 marks=pytest.mark.skipif(
+                     sys.hexversion < 0x03060000,
+                     reason="Doesn't show an error on Python 3.5")),
+    ("http://foo:80:80/monkey",
+     "Invalid port: invalid literal for int() with base 10: '80:80'"),
     ("chrome://foo:1234/bar", "Ports are unsupported with chrome scheme"),
 
     # Additional tests
     ("http://[", "Invalid IPv6 URL"),
 ])
 def test_invalid_patterns(pattern, error):
-    with pytest.raises(urlmatch.ParseError, match=error):
+    with pytest.raises(urlmatch.ParseError, match=re.escape(error)):
         urlmatch.UrlPattern(pattern)