Set user-agent for QtWebKit downloads

See #513

CHANGELOG.asciidoc

@@ -58,6 +58,7 @@ Fixed
 - Crash reports are now re-enabled when using QtWebEngine
 - Fixed crashes when closing tabs while hinting
 - Fixed starting on newer PyQt/sip versions with LibreSSL
+- When downloading files with QtWebKit, an User-Agent header is set when possible.
 
 v0.9.1
 ------

qutebrowser/browser/browsertab.py

@@ -762,6 +762,14 @@ class AbstractTab(QWidget):
         """
         raise NotImplementedError
 
+    def user_agent(self):
+        """Get the user agent for this tab.
+
+        This is only implemented for QtWebKit.
+        For QtWebEngine, always returns None.
+        """
+        raise NotImplementedError
+
     def __repr__(self):
         try:
             url = utils.elide(self.url().toDisplayString(QUrl.EncodeUnicode),

qutebrowser/browser/commands.py

@@ -1332,18 +1332,22 @@ class CommandDispatcher:
         if dest is not None:
             target = downloads.FileDownloadTarget(dest)
 
+        tab = self._current_widget()
+        user_agent = tab.user_agent()
+
         if url:
             if mhtml_:
                 raise cmdexc.CommandError("Can only download the current page"
                                           " as mhtml.")
             url = urlutils.qurl_from_user_input(url)
             urlutils.raise_cmdexc_if_invalid(url)
-            download_manager.get(url, target=target)
+            download_manager.get(url, user_agent=user_agent, target=target)
         elif mhtml_:
             self._download_mhtml(target)
         else:
-            qnam = self._current_widget().networkaccessmanager()
+            qnam = tab.networkaccessmanager()
-            download_manager.get(self._current_url(), qnam=qnam, target=target)
+            download_manager.get(self._current_url(), user_agent=user_agent,
+                                 qnam=qnam, target=target)
 
     def _download_mhtml(self, target=None):
         """Download the current page as an MHTML file, including all assets.

qutebrowser/browser/hints.py

@@ -287,11 +287,13 @@ class HintActions:
 
         prompt = False if context.rapid else None
         qnam = context.tab.networkaccessmanager()
+        user_agent = context.tab.user_agent()
 
         # FIXME:qtwebengine do this with QtWebEngine downloads?
         download_manager = objreg.get('qtnetwork-download-manager',
                                       scope='window', window=self._win_id)
-        download_manager.get(url, qnam=qnam, prompt_download_directory=prompt)
+        download_manager.get(url, qnam=qnam, user_agent=user_agent,
+                             prompt_download_directory=prompt)
 
     def call_userscript(self, elem, context):
         """Call a userscript from a hint.

qutebrowser/browser/qtnetworkdownloads.py

@@ -366,11 +366,12 @@ class DownloadManager(downloads.AbstractDownloadManager):
             win_id, None, self)
 
     @pyqtSlot('QUrl')
-    def get(self, url, **kwargs):
+    def get(self, url, *, user_agent=None, **kwargs):
         """Start a download with a link URL.
 
         Args:
             url: The URL to get, as QUrl
+            user_agent: The UA to set for the request, or None.
             **kwargs: passed to get_request().
 
         Return:
@@ -380,6 +381,8 @@ class DownloadManager(downloads.AbstractDownloadManager):
             urlutils.invalid_url_error(url, "start download")
             return
         req = QNetworkRequest(url)
+        if user_agent is not None:
+            req.setHeader(QNetworkRequest.UserAgentHeader, user_agent)
         return self.get_request(req, **kwargs)
 
     def get_request(self, request, *, target=None, **kwargs):

qutebrowser/browser/webengine/webenginetab.py

@@ -611,6 +611,9 @@ class WebEngineTab(browsertab.AbstractTab):
     def networkaccessmanager(self):
         return None
 
+    def user_agent(self):
+        return None
+
     def clear_ssl_errors(self):
         raise browsertab.UnsupportedOperationError
 

qutebrowser/browser/webkit/webkittab.py

@@ -686,6 +686,10 @@ class WebKitTab(browsertab.AbstractTab):
     def networkaccessmanager(self):
         return self._widget.page().networkAccessManager()
 
+    def user_agent(self):
+        page = self._widget.page()
+        return page.userAgentForUrl(self.url())
+
     @pyqtSlot()
     def _on_frame_load_finished(self):
         """Make sure we emit an appropriate status when loading finished.

tests/end2end/features/downloads.feature

@@ -588,3 +588,18 @@ Feature: Downloading things from a website.
        And I open stream-bytes/1024 without waiting
        And I wait until the download is finished
        Then the downloaded file 1024 should exist
+
+    @qtwebengine_skip: We can't get the UA from the page there
+    Scenario: user-agent when using :download
+        When I open user-agent
+        And I run :download
+        And I wait until the download is finished
+        Then the downloaded file user-agent should contain Safari/
+
+    @qtwebengine_skip: We can't get the UA from the page there
+    Scenario: user-agent when using hints
+        When I open /
+        And I run :hint links download
+        And I run :follow-hint d
+        And I wait until the download is finished
+        Then the downloaded file user-agent should contain Safari/

tests/end2end/features/test_downloads_bdd.py

@@ -95,6 +95,13 @@ def download_size(filename, size, tmpdir):
     assert path.size() == int(size)
 
 
+@bdd.then(bdd.parsers.parse("The downloaded file {filename} should contain "
+                            "{text}"))
+def download_contents(filename, text, tmpdir):
+    path = tmpdir / 'downloads' / filename
+    assert text in path.read()
+
+
 @bdd.then(bdd.parsers.parse('The download prompt should be shown with '
                             '"{path}"'))
 def download_prompt(tmpdir, quteproc, path):