diff --git a/qutebrowser/browser/downloads.py b/qutebrowser/browser/downloads.py
index 6d6806205..2537d0c5c 100644
--- a/qutebrowser/browser/downloads.py
+++ b/qutebrowser/browser/downloads.py
@@ -724,7 +724,7 @@ class DownloadManager(QAbstractListModel):
         if prompt_download_directory is None:
             prompt_download_directory = config.get(
                 'storage', 'prompt-download-directory')
-        if not prompt_download_directory:
+        if not prompt_download_directory and not fileobj:
             filename = config.get('storage', 'download-directory')
 
         if fileobj is not None or filename is not None:
@@ -818,7 +818,7 @@ class DownloadManager(QAbstractListModel):
 
         prompt_download_directory = config.get('storage',
                                                'prompt-download-directory')
-        if not prompt_download_directory:
+        if not prompt_download_directory and not fileobj:
             filename = config.get('storage', 'download-directory')
 
         if filename is not None: