diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py
index 5dffbfc34..df6b08a2f 100644
--- a/qutebrowser/config/config.py
+++ b/qutebrowser/config/config.py
@@ -189,6 +189,8 @@ class KeyConfig:
             raise configexc.DuplicateKeyError(key)
 
         bindings = self._config.get_obj('bindings.commands')
+        if mode not in bindings:
+            bindings[mode] = {}
         bindings[mode][key] = command
         self._config.update_mutables(save_yaml=save_yaml)
 
@@ -198,11 +200,13 @@ class KeyConfig:
 
         bindings_commands = self._config.get_obj('bindings.commands')
 
-        if key in bindings_commands:
+        if key in val.bindings.commands[mode]:
             # In custom bindings -> remove it
             del bindings_commands[mode][key]
         elif key in val.bindings.default[mode]:
             # In default bindings -> shadow it with None
+            if mode not in bindings_commands:
+                bindings_commands[mode] = {}
             bindings_commands[mode][key] = None
         else:
             raise configexc.KeybindingError("Can't find binding '{}' in section '{}'!"
diff --git a/qutebrowser/config/configtypes.py b/qutebrowser/config/configtypes.py
index b01d3971e..84ef433d3 100644
--- a/qutebrowser/config/configtypes.py
+++ b/qutebrowser/config/configtypes.py
@@ -389,18 +389,6 @@ class List(BaseType):
         self.valtype = valtype
         self.length = length
 
-    def _none_value(self, value=None):
-        """Return the value to be used when the setting is None.
-
-        Args:
-            value: An existing value to be returned.
-        """
-        if value is None:
-            return []
-        else:
-            assert value == [], value
-            return value
-
     def get_name(self):
         name = super().get_name()
         if self._show_valtype:
@@ -413,7 +401,7 @@ class List(BaseType):
     def from_str(self, value):
         self._basic_str_validation(value)
         if not value:
-            return self._none_value()
+            return None
 
         try:
             yaml_val = utils.yaml_load(value)
@@ -428,7 +416,7 @@ class List(BaseType):
     def to_py(self, value):
         self._basic_py_validation(value, list)
         if not value:
-            return self._none_value(value)
+            return []
 
         if self.length is not None and len(value) != self.length:
             raise configexc.ValidationError(value, "Exactly {} values need to "
@@ -1052,31 +1040,10 @@ class Dict(BaseType):
             raise configexc.ValidationError(
                 value, "Required keys {}".format(self.required_keys))
 
-    def _none_value(self, value=None):
-        """Return the value to be used when the setting is None.
-
-        Args:
-            value: An existing value to be returned.
-        """
-        if self.fixed_keys is None:
-            if value is None:
-                return {}
-            else:
-                assert value == {}, value
-                return value
-        else:
-            if value is None:
-                return {key: {} for key in self.fixed_keys}
-            else:
-                assert value == {}, value
-                for key in self.fixed_keys:
-                    value[key] = {}
-                return value
-
     def from_str(self, value):
         self._basic_str_validation(value)
         if not value:
-            return self._none_value()
+            return None
 
         try:
             yaml_val = utils.yaml_load(value)
@@ -1091,7 +1058,11 @@ class Dict(BaseType):
     def to_py(self, value):
         self._basic_py_validation(value, dict)
         if not value:
-            return self._none_value(value)
+            if self.fixed_keys is None:
+                return {}
+            else:
+                return {key: self.valtype.to_py(None)
+                        for key in self.fixed_keys}
 
         self._validate_keys(value)
 
@@ -1197,7 +1168,7 @@ class ShellCommand(List):
     def from_str(self, value):
         self._basic_str_validation(value)
         if not value:
-            return self._none_value()
+            return None
 
         try:
             split_val = shlex.split(value)
@@ -1316,17 +1287,13 @@ class Padding(Dict):
     _show_valtype = False
 
     def __init__(self, none_ok=False):
-        super().__init__(keytype=String(), valtype=Int(minval=0),
+        super().__init__(keytype=String(),
+                         valtype=Int(minval=0, none_ok=none_ok),
                          fixed_keys=['top', 'bottom', 'left', 'right'],
                          none_ok=none_ok)
 
-    def _none_value(self, _value=None):
-        return None
-
     def to_py(self, value):
         d = super().to_py(value)
-        if not d:
-            return None
         return PaddingValues(**d)
 
 
diff --git a/tests/unit/config/test_configtypes.py b/tests/unit/config/test_configtypes.py
index 666bf0c53..50f281c11 100644
--- a/tests/unit/config/test_configtypes.py
+++ b/tests/unit/config/test_configtypes.py
@@ -253,12 +253,16 @@ class TestAll:
     def test_none_ok_true(self, klass):
         """Test None and empty string values with none_ok=True."""
         typ = klass(none_ok=True)
-        if isinstance(typ, (configtypes.Dict, configtypes.List)):
-            expected = typ._none_value()
+        if isinstance(typ, configtypes.Padding):
+            to_py_expected = configtypes.PaddingValues(None, None, None, None)
+        elif isinstance(typ, configtypes.Dict):
+            to_py_expected = {}
+        elif isinstance(typ, configtypes.List):
+            to_py_expected = []
         else:
-            expected = None
-        assert typ.from_str('') == expected
-        assert typ.to_py(None) == expected
+            to_py_expected = None
+        assert typ.from_str('') is None
+        assert typ.to_py(None) == to_py_expected
         assert typ.to_str(None) == ''
 
     @pytest.mark.parametrize('method, value', [
@@ -575,7 +579,8 @@ class TestList:
         except configexc.ValidationError:
             pass
         else:
-            assert typ.from_str(typ.to_str(converted)) == converted
+            expected = converted if converted else None
+            assert typ.from_str(typ.to_str(converted)) == expected
 
     @hypothesis.given(val=strategies.lists(strategies.just('foo')))
     def test_hypothesis_text(self, klass, val):
@@ -1434,7 +1439,8 @@ class TestDict:
                   none_ok=True)
         try:
             converted = d.to_py(val)
-            assert d.from_str(d.to_str(converted)) == converted
+            expected = converted if converted else None
+            assert d.from_str(d.to_str(converted)) == expected
         except configexc.ValidationError:
             # Invalid unicode in the string, etc...
             hypothesis.assume(False)