diff --git a/qutebrowser/keyinput/modeman.py b/qutebrowser/keyinput/modeman.py
index e32830f50..5204c8068 100644
--- a/qutebrowser/keyinput/modeman.py
+++ b/qutebrowser/keyinput/modeman.py
@@ -267,6 +267,11 @@ class ModeManager(QObject):
             raise cmdexc.CommandError(
                 "Mode {} can't be entered manually!".format(mode))
 
+        if (config.val.tabs.mode_on_change == 'restore' and
+                m.name in ['insert', 'passthrough']):
+            window = objreg.get('main-window', scope='window',
+                                window=self._win_id)
+            window.tabbed_browser.currentWidget().data.input_mode = m
         self.enter(m, 'command')
 
     @pyqtSlot(usertypes.KeyMode, str, bool)
@@ -301,6 +306,11 @@ class ModeManager(QObject):
         """Leave the mode we're currently in."""
         if self.mode == usertypes.KeyMode.normal:
             raise ValueError("Can't leave normal mode!")
+        if config.val.tabs.mode_on_change == 'restore':
+            window = objreg.get('main-window', scope='window',
+                                window=self._win_id)
+            tab = window.tabbed_browser.currentWidget()
+            tab.data.input_mode = usertypes.KeyMode.normal
         self.leave(self.mode, 'leave current')
 
     def eventFilter(self, event):
diff --git a/qutebrowser/mainwindow/tabbedbrowser.py b/qutebrowser/mainwindow/tabbedbrowser.py
index 8cee35524..94fd05b83 100644
--- a/qutebrowser/mainwindow/tabbedbrowser.py
+++ b/qutebrowser/mainwindow/tabbedbrowser.py
@@ -647,8 +647,6 @@ class TabbedBrowser(tabwidget.TabWidget):
     def on_current_changed(self, idx):
         """Set last-focused-tab and leave hinting mode when focus changed."""
         mode_on_change = config.val.tabs.mode_on_change
-        modes_to_save = [usertypes.KeyMode.insert,
-                         usertypes.KeyMode.passthrough]
         if idx == -1 or self.shutting_down:
             # closing the last tab (before quitting) or shutting down
             return
@@ -657,23 +655,18 @@ class TabbedBrowser(tabwidget.TabWidget):
             log.webview.debug("on_current_changed got called with invalid "
                               "index {}".format(idx))
             return
-        if self._now_focused is not None and mode_on_change == 'restore':
-            current_mode = modeman.instance(self._win_id).mode
-            if current_mode not in modes_to_save:
-                current_mode = usertypes.KeyMode.normal
-            self._now_focused.data.input_mode = current_mode
 
         log.modes.debug("Current tab changed, focusing {!r}".format(tab))
         tab.setFocus()
 
         modes_to_leave = [usertypes.KeyMode.hint, usertypes.KeyMode.caret]
-        if mode_on_change != 'persist':
-            modes_to_leave += modes_to_save
+        if mode_on_change == 'normal':
+            modes_to_leave += [usertypes.KeyMode.insert,
+                               usertypes.KeyMode.passthrough]
         for mode in modes_to_leave:
             modeman.leave(self._win_id, mode, 'tab changed', maybe=True)
         if mode_on_change == 'restore':
-            modeman.enter(self._win_id, tab.data.input_mode,
-                          'restore input mode for tab')
+            modeman.enter(self._win_id, tab.data.input_mode, 'restore')
         if self._now_focused is not None:
             objreg.register('last-focused-tab', self._now_focused, update=True,
                             scope='window', window=self._win_id)