Merge branch 'Carpetsmoker-referer-header'
This commit is contained in:
Florian Bruhin
commit
9041d6bdfc
@ -27,6 +27,8 @@ Added
|
||||
the current item in the completion (for quickmarks/bookmarks).
|
||||
- New settings `tabs -> padding` and `tabs -> indicator-tabbing` to control the
|
||||
size/padding of the tabbar.
|
||||
- New setting `network -> referer-header` to configure when the referer should
|
||||
be sent (by default it's only sent while on the same domain).
|
||||
|
||||
Changed
|
||||
~~~~~~~
|
||||
|
||||
@ -136,8 +136,8 @@ Contributors, sorted by the number of commits in descending order:
|
||||
* Florian Bruhin
|
||||
* Bruno Oliveira
|
||||
* Antoni Boucher
|
||||
* Raphael Pierzina
|
||||
* Martin Tournoij
|
||||
* Raphael Pierzina
|
||||
* Joel Torstensson
|
||||
* Claude
|
||||
* Lamar Pavel
|
||||
|
||||
@ -54,6 +54,7 @@
|
||||
|Setting|Description
|
||||
|<<network-do-not-track,do-not-track>>|Value to send in the `DNT` header.
|
||||
|<<network-accept-language,accept-language>>|Value to send in the `accept-language` header.
|
||||
|<<network-referer-header,referer-header>>|Send the Referer header
|
||||
|<<network-user-agent,user-agent>>|User agent to send. Empty to send the default.
|
||||
|<<network-proxy,proxy>>|The proxy to use.
|
||||
|<<network-proxy-dns-requests,proxy-dns-requests>>|Whether to send DNS requests over the configured proxy.
|
||||
@ -640,6 +641,18 @@ Value to send in the `accept-language` header.
|
||||
|
||||
Default: +pass:[en-US,en]+
|
||||
|
||||
[[network-referer-header]]
|
||||
=== referer-header
|
||||
Send the Referer header
|
||||
|
||||
Valid values:
|
||||
|
||||
* +always+: Always send.
|
||||
* +never+: Never send; this is not recommended, as some sites may break.
|
||||
* +same-domain+: Only send for the same domain. This will still protect your privacy, but shouldn't break any sites.
|
||||
|
||||
Default: +pass:[same-domain]+
|
||||
|
||||
[[network-user-agent]]
|
||||
=== user-agent
|
||||
User agent to send. Empty to send the default.
|
||||
|
||||
@ -22,7 +22,7 @@
|
||||
import collections
|
||||
|
||||
from PyQt5.QtCore import (pyqtSlot, pyqtSignal, PYQT_VERSION, QCoreApplication,
|
||||
QUrl)
|
||||
QUrl, QByteArray)
|
||||
from PyQt5.QtNetwork import (QNetworkAccessManager, QNetworkReply, QSslError,
|
||||
QSslSocket)
|
||||
|
||||
@ -337,6 +337,21 @@ class NetworkManager(QNetworkAccessManager):
|
||||
dnt = '0'.encode('ascii')
|
||||
req.setRawHeader('DNT'.encode('ascii'), dnt)
|
||||
req.setRawHeader('X-Do-Not-Track'.encode('ascii'), dnt)
|
||||
|
||||
current_url = objreg.get('webview', scope='tab', window=self._win_id,
|
||||
tab=self._tab_id).url()
|
||||
referer_header_conf = config.get('network', 'referer-header')
|
||||
|
||||
if referer_header_conf == 'never':
|
||||
# Note: using ''.encode('ascii') sends a header with no value,
|
||||
# instead of no header at all
|
||||
req.setRawHeader('Referer'.encode('ascii'), QByteArray())
|
||||
elif (referer_header_conf== 'same-domain' and current_url.isValid() and
|
||||
not urlutils.same_domain(req.url(), current_url)):
|
||||
req.setRawHeader('Referer'.encode('ascii'), QByteArray())
|
||||
# If refer_header_conf is set to 'always', we leave the header alone as
|
||||
# QtWebKit did set it.
|
||||
|
||||
accept_language = config.get('network', 'accept-language')
|
||||
if accept_language is not None:
|
||||
req.setRawHeader('Accept-Language'.encode('ascii'),
|
||||
|
||||
@ -326,6 +326,10 @@ def data(readonly=False):
|
||||
SettingValue(typ.String(none_ok=True), 'en-US,en'),
|
||||
"Value to send in the `accept-language` header."),
|
||||
|
||||
('referer-header',
|
||||
SettingValue(typ.Referer(), 'same-domain'),
|
||||
"Send the Referer header"),
|
||||
|
||||
('user-agent',
|
||||
SettingValue(typ.UserAgent(none_ok=True), ''),
|
||||
"User agent to send. Empty to send the default."),
|
||||
|
||||
@ -1489,6 +1489,18 @@ class DownloadPathSuggestion(BaseType):
|
||||
('both', "Show download path and filename."))
|
||||
|
||||
|
||||
class Referer(BaseType):
|
||||
|
||||
"""Send the Referer header."""
|
||||
|
||||
valid_values = ValidValues(('always', "Always send."),
|
||||
('never', "Never send; this is not recommended,"
|
||||
" as some sites may break."),
|
||||
('same-domain', "Only send for the same domain."
|
||||
" This will still protect your privacy, but"
|
||||
" shouldn't break any sites."))
|
||||
|
||||
|
||||
class UserAgent(BaseType):
|
||||
|
||||
"""The user agent to use."""
|
||||
|
||||
@ -392,6 +392,36 @@ def get_errstring(url, base="Invalid URL"):
|
||||
return base
|
||||
|
||||
|
||||
def same_domain(url1, url2):
|
||||
"""Check if url1 and url2 belong to the same website.
|
||||
|
||||
This will use a "public suffix list" to determine what a "top level domain"
|
||||
is. All further domains are ignored.
|
||||
|
||||
For example example.com and www.example.com are considered the same. but
|
||||
example.co.uk and test.co.uk are not.
|
||||
|
||||
Return:
|
||||
True if the domains are the same, False otherwise.
|
||||
"""
|
||||
if not url1.isValid():
|
||||
raise ValueError(get_errstring(url1))
|
||||
if not url2.isValid():
|
||||
raise ValueError(get_errstring(url2))
|
||||
|
||||
suffix1 = url1.topLevelDomain()
|
||||
suffix2 = url2.topLevelDomain()
|
||||
if suffix1 == '':
|
||||
return url1.host() == url2.host()
|
||||
|
||||
if not suffix1 == suffix2:
|
||||
return False
|
||||
|
||||
domain1 = url1.host()[:-len(suffix1)].split('.')[-1]
|
||||
domain2 = url2.host()[:-len(suffix2)].split('.')[-1]
|
||||
return domain1 == domain2
|
||||
|
||||
|
||||
class FuzzyUrlError(Exception):
|
||||
|
||||
"""Exception raised by fuzzy_url on problems.
|
||||
|
||||
@ -497,3 +497,21 @@ def test_fuzzy_url_error(url, raising, has_err_string):
|
||||
else:
|
||||
expected_text = "Error message"
|
||||
assert str(excinfo.value) == expected_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize('are_same, url1, url2', [
|
||||
(True, 'http://example.com', 'http://www.example.com'),
|
||||
(True, 'http://bbc.co.uk', 'https://www.bbc.co.uk'),
|
||||
(True, 'http://many.levels.of.domains.example.com', 'http://www.example.com'),
|
||||
(True, 'http://idn.иком.museum', 'http://idn2.иком.museum'),
|
||||
(True, 'http://one.not_a_valid_tld', 'http://one.not_a_valid_tld'),
|
||||
|
||||
(False, 'http://bbc.co.uk', 'http://example.co.uk'),
|
||||
(False, 'https://example.kids.museum', 'http://example.kunst.museum'),
|
||||
(False, 'http://idn.иком.museum', 'http://idn.ירושלים.museum'),
|
||||
(False, 'http://one.not_a_valid_tld', 'http://two.not_a_valid_tld'),
|
||||
])
|
||||
def test_same_domain(are_same, url1, url2):
|
||||
"""Tests for same_domain."""
|
||||
assert urlutils.same_domain(QUrl(url1), QUrl(url2)) == are_same
|
||||
assert urlutils.same_domain(QUrl(url2), QUrl(url1)) == are_same
|
||||
|
||||
Loading…
Reference in New Issue
Block a user