diff --git a/qutebrowser/utils/usertypes.py b/qutebrowser/utils/usertypes.py
index a802e7244..0dd63fc4d 100644
--- a/qutebrowser/utils/usertypes.py
+++ b/qutebrowser/utils/usertypes.py
@@ -120,7 +120,7 @@ class NeighborList(collections.abc.Sequence):
             item = min(items, key=lambda tpl: abs(self.fuzzyval - tpl[1]))
         else:
             sorted_items = sorted([(idx, e) for (idx, e) in
-                                   enumerate(self.items)])
+                                   enumerate(self.items)], key=lambda e: e[1])
             idx = 0 if offset < 0 else -1
             item = sorted_items[idx]
         self._idx = item[0]