diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
index 6326e6279..548313b93 100644
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -573,13 +573,16 @@ class AbstractDownloadItem(QObject):
         """Set a temporary file when opening the download."""
         raise NotImplementedError
 
-    def _set_filename(self, filename, *, force_overwrite=False):
+    def _set_filename(self, filename, *, force_overwrite=False,
+                      remember_directory=True):
         """Set the filename to save the download to.
 
         Args:
             filename: The full filename to save the download to.
                       None: special value to stop the download.
             force_overwrite: Force overwriting existing files.
+            remember_directory: If True, remember the directory for future
+                                downloads.
         """
         global last_used_directory
         filename = os.path.expanduser(filename)
@@ -609,7 +612,8 @@ class AbstractDownloadItem(QObject):
                                                   os.path.expanduser('~'))
 
         self.basename = os.path.basename(self._filename)
-        last_used_directory = os.path.dirname(self._filename)
+        if remember_directory:
+            last_used_directory = os.path.dirname(self._filename)
 
         log.downloads.debug("Setting filename to {}".format(filename))
         if force_overwrite:
diff --git a/qutebrowser/browser/webengine/webenginedownloads.py b/qutebrowser/browser/webengine/webenginedownloads.py
index e78e939e1..44b326d36 100644
--- a/qutebrowser/browser/webengine/webenginedownloads.py
+++ b/qutebrowser/browser/webengine/webenginedownloads.py
@@ -96,7 +96,8 @@ class DownloadItem(downloads.AbstractDownloadItem):
         raise downloads.UnsupportedOperationError
 
     def _set_tempfile(self, fileobj):
-        self._set_filename(fileobj.name, force_overwrite=True)
+        self._set_filename(fileobj.name, force_overwrite=True,
+                           remember_directory=False)
 
     def _ensure_can_set_filename(self, filename):
         state = self._qt_item.state()
diff --git a/tests/end2end/features/downloads.feature b/tests/end2end/features/downloads.feature
index 5b4c4f9a4..1b0ff1170 100644
--- a/tests/end2end/features/downloads.feature
+++ b/tests/end2end/features/downloads.feature
@@ -438,6 +438,21 @@ Feature: Downloading things from a website.
         And I open data/downloads/download2.bin without waiting
         Then the download prompt should be shown with "(tmpdir)/download2.bin"
 
+    # https://github.com/The-Compiler/qutebrowser/issues/2173
+
+    Scenario: Remembering the temporary download directory (issue 2173)
+        When I set storage -> prompt-download-directory to true
+        And I set completion -> download-path-suggestion to both
+        And I set storage -> remember-download-directory to true
+        And I open data/downloads/download.bin without waiting
+        And I wait for the download prompt for "*"
+        And I run :prompt-accept (tmpdir)
+        And I open data/downloads/download.bin without waiting
+        And I wait for the download prompt for "*"
+        And I directly open the download
+        And I open data/downloads/download.bin without waiting
+        Then the download prompt should be shown with "(tmpdir)/download.bin"
+
     # Overwriting files
 
     Scenario: Not overwriting an existing file