diff --git a/qutebrowser/config/configfiles.py b/qutebrowser/config/configfiles.py
index ee22ff8e0..7a555e6a5 100644
--- a/qutebrowser/config/configfiles.py
+++ b/qutebrowser/config/configfiles.py
@@ -255,8 +255,9 @@ class YamlConfig(QObject):
 
     def unset(self, name, *, pattern=None):
         """Remove the given option name if it's configured."""
-        self._values[name].remove(pattern)
-        self._mark_changed()
+        changed = self._values[name].remove(pattern)
+        if changed:
+            self._mark_changed()
 
     def clear(self):
         """Clear all values from the YAML file."""
diff --git a/qutebrowser/config/configutils.py b/qutebrowser/config/configutils.py
index 043650524..cf5569b61 100644
--- a/qutebrowser/config/configutils.py
+++ b/qutebrowser/config/configutils.py
@@ -110,7 +110,9 @@ class Values:
     def remove(self, pattern=None):
         """Remove the value with the given pattern."""
         # FIXME:conf Should this ignore patterns which weren't found?
+        old_len = len(self._values)
         self._values = [v for v in self._values if v.pattern != pattern]
+        return old_len != len(self._values)
 
     def clear(self):
         """Clear all customization for this value."""