Merge branch 'Carpetsmoker-referer-header'

CHANGELOG.asciidoc

@@ -27,6 +27,8 @@ Added
   the current item in the completion (for quickmarks/bookmarks).
 - New settings `tabs -> padding` and `tabs -> indicator-tabbing` to control the
   size/padding of the tabbar.
+- New setting `network -> referer-header` to configure when the referer should
+  be sent (by default it's only sent while on the same domain).
 
 Changed
 ~~~~~~~

README.asciidoc

@@ -136,8 +136,8 @@ Contributors, sorted by the number of commits in descending order:
 * Florian Bruhin
 * Bruno Oliveira
 * Antoni Boucher
-* Raphael Pierzina
 * Martin Tournoij
+* Raphael Pierzina
 * Joel Torstensson
 * Claude
 * Lamar Pavel

doc/help/settings.asciidoc

@@ -54,6 +54,7 @@
 |Setting|Description
 |<<network-do-not-track,do-not-track>>|Value to send in the `DNT` header.
 |<<network-accept-language,accept-language>>|Value to send in the `accept-language` header.
+|<<network-referer-header,referer-header>>|Send the Referer header
 |<<network-user-agent,user-agent>>|User agent to send. Empty to send the default.
 |<<network-proxy,proxy>>|The proxy to use.
 |<<network-proxy-dns-requests,proxy-dns-requests>>|Whether to send DNS requests over the configured proxy.
@@ -640,6 +641,18 @@ Value to send in the `accept-language` header.
 
 Default: +pass:[en-US,en]+
 
+[[network-referer-header]]
+=== referer-header
+Send the Referer header
+
+Valid values:
+
+ * +always+: Always send.
+ * +never+: Never send; this is not recommended, as some sites may break.
+ * +same-domain+: Only send for the same domain. This will still protect your privacy, but shouldn't break any sites.
+
+Default: +pass:[same-domain]+
+
 [[network-user-agent]]
 === user-agent
 User agent to send. Empty to send the default.

qutebrowser/browser/network/networkmanager.py

@@ -22,7 +22,7 @@
 import collections
 
 from PyQt5.QtCore import (pyqtSlot, pyqtSignal, PYQT_VERSION, QCoreApplication,
-                          QUrl)
+                          QUrl, QByteArray)
 from PyQt5.QtNetwork import (QNetworkAccessManager, QNetworkReply, QSslError,
                              QSslSocket)
 
@@ -337,6 +337,21 @@ class NetworkManager(QNetworkAccessManager):
             dnt = '0'.encode('ascii')
         req.setRawHeader('DNT'.encode('ascii'), dnt)
         req.setRawHeader('X-Do-Not-Track'.encode('ascii'), dnt)
+
+        current_url = objreg.get('webview', scope='tab', window=self._win_id,
+                                 tab=self._tab_id).url()
+        referer_header_conf = config.get('network', 'referer-header')
+
+        if referer_header_conf == 'never':
+            # Note: using ''.encode('ascii') sends a header with no value,
+            # instead of no header at all
+            req.setRawHeader('Referer'.encode('ascii'), QByteArray())
+        elif (referer_header_conf== 'same-domain' and current_url.isValid() and
+                not urlutils.same_domain(req.url(), current_url)):
+            req.setRawHeader('Referer'.encode('ascii'), QByteArray())
+        # If refer_header_conf is set to 'always', we leave the header alone as
+        # QtWebKit did set it.
+
         accept_language = config.get('network', 'accept-language')
         if accept_language is not None:
             req.setRawHeader('Accept-Language'.encode('ascii'),

qutebrowser/config/configdata.py

@@ -326,6 +326,10 @@ def data(readonly=False):
              SettingValue(typ.String(none_ok=True), 'en-US,en'),
              "Value to send in the `accept-language` header."),
 
+            ('referer-header',
+             SettingValue(typ.Referer(), 'same-domain'),
+             "Send the Referer header"),
+
             ('user-agent',
              SettingValue(typ.UserAgent(none_ok=True), ''),
              "User agent to send. Empty to send the default."),

qutebrowser/config/configtypes.py

@@ -1489,6 +1489,18 @@ class DownloadPathSuggestion(BaseType):
                                ('both', "Show download path and filename."))
 
 
+class Referer(BaseType):
+
+    """Send the Referer header."""
+
+    valid_values = ValidValues(('always', "Always send."),
+                               ('never', "Never send; this is not recommended,"
+                                   " as some sites may break."),
+                               ('same-domain', "Only send for the same domain."
+                                   " This will still protect your privacy, but"
+                                   " shouldn't break any sites."))
+
+
 class UserAgent(BaseType):
 
     """The user agent to use."""

qutebrowser/utils/urlutils.py

@@ -392,6 +392,36 @@ def get_errstring(url, base="Invalid URL"):
         return base
 
 
+def same_domain(url1, url2):
+    """Check if url1 and url2 belong to the same website.
+
+    This will use a "public suffix list" to determine what a "top level domain"
+    is. All further domains are ignored.
+
+    For example example.com and www.example.com are considered the same. but
+    example.co.uk and test.co.uk are not.
+
+    Return:
+        True if the domains are the same, False otherwise.
+    """
+    if not url1.isValid():
+        raise ValueError(get_errstring(url1))
+    if not url2.isValid():
+        raise ValueError(get_errstring(url2))
+
+    suffix1 = url1.topLevelDomain()
+    suffix2 = url2.topLevelDomain()
+    if suffix1 == '':
+        return url1.host() == url2.host()
+
+    if not suffix1 == suffix2:
+        return False
+
+    domain1 = url1.host()[:-len(suffix1)].split('.')[-1]
+    domain2 = url2.host()[:-len(suffix2)].split('.')[-1]
+    return domain1 == domain2
+
+
 class FuzzyUrlError(Exception):
 
     """Exception raised by fuzzy_url on problems.

tests/utils/test_urlutils.py

@@ -497,3 +497,21 @@ def test_fuzzy_url_error(url, raising, has_err_string):
         else:
             expected_text = "Error message"
         assert str(excinfo.value) == expected_text
+
+
+@pytest.mark.parametrize('are_same, url1, url2', [
+    (True, 'http://example.com', 'http://www.example.com'),
+    (True, 'http://bbc.co.uk', 'https://www.bbc.co.uk'),
+    (True, 'http://many.levels.of.domains.example.com', 'http://www.example.com'),
+    (True, 'http://idn.иком.museum', 'http://idn2.иком.museum'),
+    (True, 'http://one.not_a_valid_tld', 'http://one.not_a_valid_tld'),
+
+    (False, 'http://bbc.co.uk', 'http://example.co.uk'),
+    (False, 'https://example.kids.museum', 'http://example.kunst.museum'),
+    (False, 'http://idn.иком.museum', 'http://idn.ירושלים.museum'),
+    (False, 'http://one.not_a_valid_tld', 'http://two.not_a_valid_tld'),
+])
+def test_same_domain(are_same, url1, url2):
+    """Tests for same_domain."""
+    assert urlutils.same_domain(QUrl(url1), QUrl(url2)) == are_same
+    assert urlutils.same_domain(QUrl(url2), QUrl(url1)) == are_same