diff --git a/qutebrowser/browser/browsertab.py b/qutebrowser/browser/browsertab.py
index 6ed5afe52..84e4838a7 100644
--- a/qutebrowser/browser/browsertab.py
+++ b/qutebrowser/browser/browsertab.py
@@ -342,7 +342,7 @@ class AbstractCaret(QObject):
     def _on_mode_entered(self, mode):
         raise NotImplementedError
 
-    def _on_mode_left(self):
+    def _on_mode_left(self, mode):
         raise NotImplementedError
 
     def move_to_next_line(self, count=1):
diff --git a/qutebrowser/browser/webengine/webenginetab.py b/qutebrowser/browser/webengine/webenginetab.py
index ed6697f03..4595f7a6e 100644
--- a/qutebrowser/browser/webengine/webenginetab.py
+++ b/qutebrowser/browser/webengine/webenginetab.py
@@ -223,7 +223,10 @@ class WebEngineCaret(browsertab.AbstractCaret):
         self._js_call('setInitialCursor')
 
     @pyqtSlot(usertypes.KeyMode)
-    def _on_mode_left(self):
+    def _on_mode_left(self, mode):
+        if mode != usertypes.KeyMode.caret:
+            return
+
         self.drop_selection()
         self._js_call('disableCaret')
 
diff --git a/qutebrowser/browser/webkit/webkittab.py b/qutebrowser/browser/webkit/webkittab.py
index 9395630db..aa3f5363e 100644
--- a/qutebrowser/browser/webkit/webkittab.py
+++ b/qutebrowser/browser/webkit/webkittab.py
@@ -205,8 +205,8 @@ class WebKitCaret(browsertab.AbstractCaret):
                 self._widget.page().currentFrame().evaluateJavaScript(
                     utils.read_file('javascript/position_caret.js'))
 
-    @pyqtSlot()
-    def _on_mode_left(self):
+    @pyqtSlot(usertypes.KeyMode)
+    def _on_mode_left(self, _mode):
         settings = self._widget.settings()
         if settings.testAttribute(QWebSettings.CaretBrowsingEnabled):
             if self.selection_enabled and self._widget.hasSelection():