Add a log-requests debug-flag

doc/changelog.asciidoc

@@ -15,6 +15,15 @@ breaking changes (such as renamed commands) can happen in minor releases.
 // `Fixed` for any bug fixes.
 // `Security` to invite users to upgrade in case of vulnerabilities.
 
+v1.4.0 (unreleased)
+-------------------
+
+Added
+~~~~~
+
+- New `--debug-flag log-requests` to log requests to the debug log for
+  debugging.
+
 v1.3.0
 ------
 

qutebrowser/browser/webengine/interceptor.py

@@ -19,20 +19,22 @@
 
 """A request interceptor taking care of adblocking and custom headers."""
 
-from PyQt5.QtWebEngineCore import QWebEngineUrlRequestInterceptor
+from PyQt5.QtWebEngineCore import (QWebEngineUrlRequestInterceptor,
+                                   QWebEngineUrlRequestInfo)
 
 from qutebrowser.config import config
 from qutebrowser.browser import shared
-from qutebrowser.utils import utils, log
+from qutebrowser.utils import utils, log, debug
 
 
 class RequestInterceptor(QWebEngineUrlRequestInterceptor):
 
     """Handle ad blocking and custom headers."""
 
-    def __init__(self, host_blocker, parent=None):
+    def __init__(self, host_blocker, args, parent=None):
         super().__init__(parent)
         self._host_blocker = host_blocker
+        self._args = args
 
     def install(self, profile):
         """Install the interceptor on the given QWebEngineProfile."""
@@ -54,6 +56,18 @@ class RequestInterceptor(QWebEngineUrlRequestInterceptor):
         Args:
             info: QWebEngineUrlRequestInfo &info
         """
+        if 'log-requests' in self._args.debug_flags:
+            resource_type = debug.qenum_key(QWebEngineUrlRequestInfo,
+                                            info.resourceType())
+            navigation_type = debug.qenum_key(QWebEngineUrlRequestInfo,
+                                              info.navigationType())
+            log.webview.debug("{} {}, first-party {}, resource {}, "
+                              "navigation {}".format(
+                                  bytes(info.requestMethod()).decode('ascii'),
+                                  info.requestUrl().toDisplayString(),
+                                  info.firstPartyUrl().toDisplayString(),
+                                  resource_type, navigation_type))
+
         # FIXME:qtwebengine only block ads for NavigationTypeOther?
         if self._host_blocker.is_blocked(info.requestUrl()):
             log.webview.info("Request to {} blocked by host blocker.".format(

qutebrowser/browser/webengine/webenginetab.py

@@ -62,8 +62,9 @@ def init():
 
     log.init.debug("Initializing request interceptor...")
     host_blocker = objreg.get('host-blocker')
+    args = objreg.get('args')
     req_interceptor = interceptor.RequestInterceptor(
-        host_blocker, parent=app)
+        host_blocker, args=args, parent=app)
     req_interceptor.install(webenginesettings.default_profile)
     req_interceptor.install(webenginesettings.private_profile)
 

qutebrowser/browser/webkit/network/networkmanager.py

@@ -28,7 +28,8 @@ from PyQt5.QtCore import (pyqtSlot, pyqtSignal, QCoreApplication, QUrl,
 from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkReply, QSslSocket
 
 from qutebrowser.config import config
-from qutebrowser.utils import message, log, usertypes, utils, objreg, urlutils
+from qutebrowser.utils import (message, log, usertypes, utils, objreg,
+                               urlutils, debug)
 from qutebrowser.browser import shared
 from qutebrowser.browser.webkit import certificateerror
 from qutebrowser.browser.webkit.network import (webkitqutescheme, networkreply,
@@ -147,6 +148,7 @@ class NetworkManager(QNetworkAccessManager):
             super().__init__(parent)
         log.init.debug("NetworkManager init done")
         self.adopted_downloads = 0
+        self._args = objreg.get('args')
         self._win_id = win_id
         self._tab_id = tab_id
         self._private = private
@@ -406,5 +408,13 @@ class NetworkManager(QNetworkAccessManager):
                 # the webpage shutdown here.
                 current_url = QUrl()
 
+        if 'log-requests' in self._args.debug_flags:
+            operation = debug.qenum_key(QNetworkAccessManager, op)
+            operation = operation.replace('Operation', '').upper()
+            log.webview.debug("{} {}, first-party {}".format(
+                operation,
+                req.url().toDisplayString(),
+                current_url.toDisplayString()))
+
         self.set_referer(req, current_url)
         return super().createRequest(op, req, outgoing_data)

qutebrowser/qutebrowser.py

@@ -159,9 +159,12 @@ def debug_flag_error(flag):
     Available flags:
         debug-exit: Turn on debugging of late exit.
         pdb-postmortem: Drop into pdb on exceptions.
+        no-sql-history: Don't store history items.
+        no-scroll-filtering: Process all scrolling updates.
+        log-requests: Log all network requests.
     """
     valid_flags = ['debug-exit', 'pdb-postmortem', 'no-sql-history',
-                   'no-scroll-filtering']
+                   'no-scroll-filtering', 'log-requests']
 
     if flag in valid_flags:
         return flag

tests/unit/browser/webkit/network/test_networkmanager.py

@@ -26,7 +26,7 @@ from qutebrowser.browser.webkit import cookies
 pytestmark = pytest.mark.usefixtures('cookiejar_and_cache')
 
 
-def test_init_with_private_mode():
+def test_init_with_private_mode(fake_args):
     nam = networkmanager.NetworkManager(win_id=0, tab_id=0, private=True)
     assert isinstance(nam.cookieJar(), cookies.RAMCookieJar)
     assert nam.cache() is None

tests/unit/browser/webkit/test_downloads.py

@@ -22,7 +22,8 @@ import pytest
 from qutebrowser.browser import downloads, qtnetworkdownloads
 
 
-def test_download_model(qapp, qtmodeltester, config_stub, cookiejar_and_cache):
+def test_download_model(qapp, qtmodeltester, config_stub, cookiejar_and_cache,
+                        fake_args):
     """Simple check for download model internals."""
     manager = qtnetworkdownloads.DownloadManager()
     model = downloads.DownloadModel(manager)