diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 3adf08d9a..80c74567a 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -72,6 +72,8 @@ Fixed
 - Fixed some directory browser issues on Windows
 - Fixed crash when closing a window with a finished download and delayed
   `remove-finished-downloads` setting.
+- Fixed crash when hitting `<Tab>` then `<Ctrl-C>` on pages without keyboard
+  focus.
 
 Removed
 ~~~~~~~
diff --git a/qutebrowser/browser/downloadview.py b/qutebrowser/browser/downloadview.py
index e2df65aa6..860f4f1e6 100644
--- a/qutebrowser/browser/downloadview.py
+++ b/qutebrowser/browser/downloadview.py
@@ -79,6 +79,7 @@ class DownloadView(QListView):
         self.setResizeMode(QListView.Adjust)
         self.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
         self.setSizePolicy(QSizePolicy.MinimumExpanding, QSizePolicy.Fixed)
+        self.setFocusPolicy(Qt.NoFocus)
         self.setFlow(QListView.LeftToRight)
         self.setSpacing(1)
         self._menu = None
diff --git a/qutebrowser/keyinput/modeman.py b/qutebrowser/keyinput/modeman.py
index 21c579761..fb948ec18 100644
--- a/qutebrowser/keyinput/modeman.py
+++ b/qutebrowser/keyinput/modeman.py
@@ -23,7 +23,6 @@ import functools
 
 from PyQt5.QtCore import pyqtSignal, Qt, QObject, QEvent
 from PyQt5.QtWidgets import QApplication
-from PyQt5.QtWebKitWidgets import QWebView
 
 from qutebrowser.keyinput import modeparsers, keyparser
 from qutebrowser.config import config
@@ -171,13 +170,9 @@ class ModeManager(QObject):
         is_non_alnum = (
             event.modifiers() not in (Qt.NoModifier, Qt.ShiftModifier) or
             not event.text().strip())
-        focus_widget = QApplication.instance().focusWidget()
-        is_tab = event.key() in (Qt.Key_Tab, Qt.Key_Backtab)
 
         if handled:
             filter_this = True
-        elif is_tab and not isinstance(focus_widget, QWebView):
-            filter_this = True
         elif (parser.passthrough or
                 self._forward_unbound_keys == 'all' or
                 (self._forward_unbound_keys == 'auto' and is_non_alnum)):
@@ -189,11 +184,12 @@ class ModeManager(QObject):
             self._releaseevents_to_pass.add(KeyEvent(event))
 
         if curmode != usertypes.KeyMode.insert:
+            focus_widget = QApplication.instance().focusWidget()
             log.modes.debug("handled: {}, forward-unbound-keys: {}, "
-                            "passthrough: {}, is_non_alnum: {}, is_tab {} --> "
+                            "passthrough: {}, is_non_alnum: {} --> "
                             "filter: {} (focused: {!r})".format(
                                 handled, self._forward_unbound_keys,
-                                parser.passthrough, is_non_alnum, is_tab,
+                                parser.passthrough, is_non_alnum,
                                 filter_this, focus_widget))
         return filter_this
 
diff --git a/tests/integration/features/misc.feature b/tests/integration/features/misc.feature
index c838323a1..7ac294d70 100644
--- a/tests/integration/features/misc.feature
+++ b/tests/integration/features/misc.feature
@@ -344,3 +344,20 @@ Feature: Various utility commands.
     Scenario: Running :pyeval with --quiet
         When I run :debug-pyeval --quiet 1+1
         Then "pyeval output: 2" should be logged
+        
+    ## https://github.com/The-Compiler/qutebrowser/issues/504
+
+    Scenario: Focusing download widget via Tab
+        When I open about:blank
+        And I press the key "<Tab>"
+        And I press the key "<Ctrl-C>"
+        Then no crash should happen
+
+    Scenario: Focusing download widget via Tab (original issue)
+        When I open data/prompt/jsprompt.html
+        And I run :hint
+        And I run :follow-hint a
+        And I wait for "Entering mode KeyMode.prompt *" in the log
+        And I press the key "<Tab>"
+        And I press the key "<Ctrl-C>"
+        Then no crash should happen