diff --git a/qutebrowser/config/configfiles.py b/qutebrowser/config/configfiles.py
index 2ee973730..55c6d7e3a 100644
--- a/qutebrowser/config/configfiles.py
+++ b/qutebrowser/config/configfiles.py
@@ -214,8 +214,13 @@ class ConfigAPI:
             self._keyconfig.unbind(key, mode=mode)
 
 
-def read_config_py(filename=None):
-    """Read a config.py file."""
+def read_config_py(filename=None, raising=False):
+    """Read a config.py file.
+
+    Arguments;
+        raising: Raise exceptions happening in config.py.
+                 This is needed during tests to use pytest's inspection.
+    """
     api = ConfigAPI(config.instance, config.key_instance)
 
     if filename is None:
@@ -261,6 +266,8 @@ def read_config_py(filename=None):
 
             exec(code, module.__dict__)
     except Exception as e:
+        if raising:
+            raise
         api.errors.append(configexc.ConfigErrorDesc(
             "Unhandled exception",
             exception=e, traceback=traceback.format_exc()))
diff --git a/tests/unit/config/test_configfiles.py b/tests/unit/config/test_configfiles.py
index 4f1aba9d4..5661c3ff9 100644
--- a/tests/unit/config/test_configfiles.py
+++ b/tests/unit/config/test_configfiles.py
@@ -222,7 +222,7 @@ class ConfPy:
 
     def read(self, error=False):
         """Read the config.py via configfiles and check for errors."""
-        api = configfiles.read_config_py(self.filename)
+        api = configfiles.read_config_py(self.filename, raising=not error)
         assert len(api.errors) == (1 if error else 0)
         return api
 
@@ -309,9 +309,8 @@ class TestConfigPy:
     def test_assertions(self, confpy):
         """Make sure assertions in config.py work for these tests."""
         confpy.write('assert False')
-        api = confpy.read(error=True)
-        error = api.errors[0]
-        assert isinstance(error.exception, AssertionError)
+        with pytest.raises(AssertionError):
+            confpy.read()  # no errors=True so it gets raised
 
     @pytest.mark.parametrize('line', [
         'c.colors.hints.bg = "red"',