diff --git a/qutebrowser/mainwindow/prompt.py b/qutebrowser/mainwindow/prompt.py
index 2bcfef61f..aa4fbcfd2 100644
--- a/qutebrowser/mainwindow/prompt.py
+++ b/qutebrowser/mainwindow/prompt.py
@@ -280,6 +280,12 @@ class PromptContainer(QWidget):
         Args:
             question: A Question object or None.
         """
+        item = self._layout.takeAt(0)
+        if item is not None:
+            widget = item.widget()
+            widget.hide()
+            widget.deleteLater()
+
         if question is None:
             self._prompt = None
             self.hide()