diff --git a/qutebrowser/javascript/webengine_caret.js b/qutebrowser/javascript/webengine_caret.js
index 4c2771451..79fb01dcb 100644
--- a/qutebrowser/javascript/webengine_caret.js
+++ b/qutebrowser/javascript/webengine_caret.js
@@ -865,7 +865,8 @@ window._qutebrowser.caret = (function() { // eslint-disable-line max-statements,
 
         if (CaretBrowsing.isWindows &&
                 (direction === "forward" ||
-                    direction === "right")) {
+                    direction === "right") &&
+                granularity === "word") {
             CaretBrowsing.move("left", "character");
         } else {
             window.setTimeout(() => {