diff --git a/qutebrowser/config/configfiles.py b/qutebrowser/config/configfiles.py
index efa984f77..1dd50d470 100644
--- a/qutebrowser/config/configfiles.py
+++ b/qutebrowser/config/configfiles.py
@@ -104,11 +104,11 @@ class YamlConfig:
         except KeyError:
             desc = configexc.ConfigErrorDesc(
                 "While loading data",
-                "Toplevel object does not contain 'global' key.")
+                "Toplevel object does not contain 'global' key")
             raise configexc.ConfigFileErrors('autoconfig.yml', [desc])
         except TypeError:
             desc = configexc.ConfigErrorDesc("While loading data",
-                                             "Toplevel object is not a dict.")
+                                             "Toplevel object is not a dict")
             raise configexc.ConfigFileErrors('autoconfig.yml', [desc])
 
         if not isinstance(global_obj, dict):
diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py
index 6d5f34d53..b44fd0746 100644
--- a/tests/unit/config/test_config.py
+++ b/tests/unit/config/test_config.py
@@ -880,18 +880,22 @@ def init_patch(qapp, fake_save_manager, monkeypatch, config_tmpdir,
             pass
 
 
-@pytest.mark.parametrize('load_autoconfig', [True, False])
+@pytest.mark.parametrize('load_autoconfig', [True, False])  # noqa
 @pytest.mark.parametrize('config_py', [True, 'error', False])
-@pytest.mark.parametrize('invalid_yaml', [True, False])
+@pytest.mark.parametrize('invalid_yaml', ['42', 'unknown', False])
 def test_init(init_patch, fake_save_manager, config_tmpdir, mocker,
               load_autoconfig, config_py, invalid_yaml):
     # Prepare files
     autoconfig_file = config_tmpdir / 'autoconfig.yml'
     config_py_file = config_tmpdir / 'config.py'
 
-    if invalid_yaml:
+    if invalid_yaml == '42':
         autoconfig_file.write_text('42', 'utf-8', ensure=True)
+    elif invalid_yaml == 'unknown':
+        autoconfig_file.write_text('global:\n  colors.foobar: magenta\n',
+                                   'utf-8', ensure=True)
     else:
+        assert not invalid_yaml
         autoconfig_file.write_text('global:\n  colors.hints.fg: magenta\n',
                                    'utf-8', ensure=True)
 
diff --git a/tests/unit/config/test_configfiles.py b/tests/unit/config/test_configfiles.py
index 8fdc9a3ea..2e3d88c61 100644
--- a/tests/unit/config/test_configfiles.py
+++ b/tests/unit/config/test_configfiles.py
@@ -18,6 +18,7 @@
 
 """Tests for qutebrowser.config.configfiles."""
 
+import os
 import sys
 
 import pytest
@@ -91,6 +92,49 @@ def test_yaml_config(fake_save_manager, config_tmpdir, old_config, insert):
         assert '  tabs.show: never' in lines
 
 
+@pytest.mark.parametrize('line, text, exception', [
+    ('%', 'While parsing', 'while scanning a directive'),
+    ('global: 42', 'While loading data', "'global' object is not a dict"),
+    ('foo: 42', 'While loading data',
+     "Toplevel object does not contain 'global' key"),
+    ('42', 'While loading data', "Toplevel object is not a dict"),
+])
+def test_yaml_config_invalid(fake_save_manager, config_tmpdir,
+                             line, text, exception):
+    autoconfig = config_tmpdir / 'autoconfig.yml'
+    autoconfig.write_text(line, 'utf-8', ensure=True)
+
+    yaml = configfiles.YamlConfig()
+
+    with pytest.raises(configexc.ConfigFileErrors) as excinfo:
+        yaml.load()
+
+    assert len(excinfo.value.errors) == 1
+    error = excinfo.value.errors[0]
+    assert error.text == text
+    assert str(error.exception).splitlines()[0] == exception
+    assert error.traceback is None
+
+
+def test_yaml_oserror(fake_save_manager, config_tmpdir):
+    autoconfig = config_tmpdir / 'autoconfig.yml'
+    autoconfig.ensure()
+    autoconfig.chmod(0)
+    if os.access(str(autoconfig), os.R_OK):
+        # Docker container or similar
+        pytest.skip("File was still readable")
+
+    yaml = configfiles.YamlConfig()
+    with pytest.raises(configexc.ConfigFileErrors) as excinfo:
+        yaml.load()
+
+    assert len(excinfo.value.errors) == 1
+    error = excinfo.value.errors[0]
+    assert error.text == "While reading"
+    assert isinstance(error.exception, OSError)
+    assert error.traceback is None
+
+
 class TestConfigPy:
 
     """Tests for ConfigAPI and read_config_py()."""