Merge branch 'Carpetsmoker-referer-header'
This commit is contained in:
commit
9041d6bdfc
@ -27,6 +27,8 @@ Added
|
|||||||
the current item in the completion (for quickmarks/bookmarks).
|
the current item in the completion (for quickmarks/bookmarks).
|
||||||
- New settings `tabs -> padding` and `tabs -> indicator-tabbing` to control the
|
- New settings `tabs -> padding` and `tabs -> indicator-tabbing` to control the
|
||||||
size/padding of the tabbar.
|
size/padding of the tabbar.
|
||||||
|
- New setting `network -> referer-header` to configure when the referer should
|
||||||
|
be sent (by default it's only sent while on the same domain).
|
||||||
|
|
||||||
Changed
|
Changed
|
||||||
~~~~~~~
|
~~~~~~~
|
||||||
|
@ -136,8 +136,8 @@ Contributors, sorted by the number of commits in descending order:
|
|||||||
* Florian Bruhin
|
* Florian Bruhin
|
||||||
* Bruno Oliveira
|
* Bruno Oliveira
|
||||||
* Antoni Boucher
|
* Antoni Boucher
|
||||||
* Raphael Pierzina
|
|
||||||
* Martin Tournoij
|
* Martin Tournoij
|
||||||
|
* Raphael Pierzina
|
||||||
* Joel Torstensson
|
* Joel Torstensson
|
||||||
* Claude
|
* Claude
|
||||||
* Lamar Pavel
|
* Lamar Pavel
|
||||||
|
@ -54,6 +54,7 @@
|
|||||||
|Setting|Description
|
|Setting|Description
|
||||||
|<<network-do-not-track,do-not-track>>|Value to send in the `DNT` header.
|
|<<network-do-not-track,do-not-track>>|Value to send in the `DNT` header.
|
||||||
|<<network-accept-language,accept-language>>|Value to send in the `accept-language` header.
|
|<<network-accept-language,accept-language>>|Value to send in the `accept-language` header.
|
||||||
|
|<<network-referer-header,referer-header>>|Send the Referer header
|
||||||
|<<network-user-agent,user-agent>>|User agent to send. Empty to send the default.
|
|<<network-user-agent,user-agent>>|User agent to send. Empty to send the default.
|
||||||
|<<network-proxy,proxy>>|The proxy to use.
|
|<<network-proxy,proxy>>|The proxy to use.
|
||||||
|<<network-proxy-dns-requests,proxy-dns-requests>>|Whether to send DNS requests over the configured proxy.
|
|<<network-proxy-dns-requests,proxy-dns-requests>>|Whether to send DNS requests over the configured proxy.
|
||||||
@ -640,6 +641,18 @@ Value to send in the `accept-language` header.
|
|||||||
|
|
||||||
Default: +pass:[en-US,en]+
|
Default: +pass:[en-US,en]+
|
||||||
|
|
||||||
|
[[network-referer-header]]
|
||||||
|
=== referer-header
|
||||||
|
Send the Referer header
|
||||||
|
|
||||||
|
Valid values:
|
||||||
|
|
||||||
|
* +always+: Always send.
|
||||||
|
* +never+: Never send; this is not recommended, as some sites may break.
|
||||||
|
* +same-domain+: Only send for the same domain. This will still protect your privacy, but shouldn't break any sites.
|
||||||
|
|
||||||
|
Default: +pass:[same-domain]+
|
||||||
|
|
||||||
[[network-user-agent]]
|
[[network-user-agent]]
|
||||||
=== user-agent
|
=== user-agent
|
||||||
User agent to send. Empty to send the default.
|
User agent to send. Empty to send the default.
|
||||||
|
@ -22,7 +22,7 @@
|
|||||||
import collections
|
import collections
|
||||||
|
|
||||||
from PyQt5.QtCore import (pyqtSlot, pyqtSignal, PYQT_VERSION, QCoreApplication,
|
from PyQt5.QtCore import (pyqtSlot, pyqtSignal, PYQT_VERSION, QCoreApplication,
|
||||||
QUrl)
|
QUrl, QByteArray)
|
||||||
from PyQt5.QtNetwork import (QNetworkAccessManager, QNetworkReply, QSslError,
|
from PyQt5.QtNetwork import (QNetworkAccessManager, QNetworkReply, QSslError,
|
||||||
QSslSocket)
|
QSslSocket)
|
||||||
|
|
||||||
@ -337,6 +337,21 @@ class NetworkManager(QNetworkAccessManager):
|
|||||||
dnt = '0'.encode('ascii')
|
dnt = '0'.encode('ascii')
|
||||||
req.setRawHeader('DNT'.encode('ascii'), dnt)
|
req.setRawHeader('DNT'.encode('ascii'), dnt)
|
||||||
req.setRawHeader('X-Do-Not-Track'.encode('ascii'), dnt)
|
req.setRawHeader('X-Do-Not-Track'.encode('ascii'), dnt)
|
||||||
|
|
||||||
|
current_url = objreg.get('webview', scope='tab', window=self._win_id,
|
||||||
|
tab=self._tab_id).url()
|
||||||
|
referer_header_conf = config.get('network', 'referer-header')
|
||||||
|
|
||||||
|
if referer_header_conf == 'never':
|
||||||
|
# Note: using ''.encode('ascii') sends a header with no value,
|
||||||
|
# instead of no header at all
|
||||||
|
req.setRawHeader('Referer'.encode('ascii'), QByteArray())
|
||||||
|
elif (referer_header_conf== 'same-domain' and current_url.isValid() and
|
||||||
|
not urlutils.same_domain(req.url(), current_url)):
|
||||||
|
req.setRawHeader('Referer'.encode('ascii'), QByteArray())
|
||||||
|
# If refer_header_conf is set to 'always', we leave the header alone as
|
||||||
|
# QtWebKit did set it.
|
||||||
|
|
||||||
accept_language = config.get('network', 'accept-language')
|
accept_language = config.get('network', 'accept-language')
|
||||||
if accept_language is not None:
|
if accept_language is not None:
|
||||||
req.setRawHeader('Accept-Language'.encode('ascii'),
|
req.setRawHeader('Accept-Language'.encode('ascii'),
|
||||||
|
@ -326,6 +326,10 @@ def data(readonly=False):
|
|||||||
SettingValue(typ.String(none_ok=True), 'en-US,en'),
|
SettingValue(typ.String(none_ok=True), 'en-US,en'),
|
||||||
"Value to send in the `accept-language` header."),
|
"Value to send in the `accept-language` header."),
|
||||||
|
|
||||||
|
('referer-header',
|
||||||
|
SettingValue(typ.Referer(), 'same-domain'),
|
||||||
|
"Send the Referer header"),
|
||||||
|
|
||||||
('user-agent',
|
('user-agent',
|
||||||
SettingValue(typ.UserAgent(none_ok=True), ''),
|
SettingValue(typ.UserAgent(none_ok=True), ''),
|
||||||
"User agent to send. Empty to send the default."),
|
"User agent to send. Empty to send the default."),
|
||||||
|
@ -1489,6 +1489,18 @@ class DownloadPathSuggestion(BaseType):
|
|||||||
('both', "Show download path and filename."))
|
('both', "Show download path and filename."))
|
||||||
|
|
||||||
|
|
||||||
|
class Referer(BaseType):
|
||||||
|
|
||||||
|
"""Send the Referer header."""
|
||||||
|
|
||||||
|
valid_values = ValidValues(('always', "Always send."),
|
||||||
|
('never', "Never send; this is not recommended,"
|
||||||
|
" as some sites may break."),
|
||||||
|
('same-domain', "Only send for the same domain."
|
||||||
|
" This will still protect your privacy, but"
|
||||||
|
" shouldn't break any sites."))
|
||||||
|
|
||||||
|
|
||||||
class UserAgent(BaseType):
|
class UserAgent(BaseType):
|
||||||
|
|
||||||
"""The user agent to use."""
|
"""The user agent to use."""
|
||||||
|
@ -392,6 +392,36 @@ def get_errstring(url, base="Invalid URL"):
|
|||||||
return base
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def same_domain(url1, url2):
|
||||||
|
"""Check if url1 and url2 belong to the same website.
|
||||||
|
|
||||||
|
This will use a "public suffix list" to determine what a "top level domain"
|
||||||
|
is. All further domains are ignored.
|
||||||
|
|
||||||
|
For example example.com and www.example.com are considered the same. but
|
||||||
|
example.co.uk and test.co.uk are not.
|
||||||
|
|
||||||
|
Return:
|
||||||
|
True if the domains are the same, False otherwise.
|
||||||
|
"""
|
||||||
|
if not url1.isValid():
|
||||||
|
raise ValueError(get_errstring(url1))
|
||||||
|
if not url2.isValid():
|
||||||
|
raise ValueError(get_errstring(url2))
|
||||||
|
|
||||||
|
suffix1 = url1.topLevelDomain()
|
||||||
|
suffix2 = url2.topLevelDomain()
|
||||||
|
if suffix1 == '':
|
||||||
|
return url1.host() == url2.host()
|
||||||
|
|
||||||
|
if not suffix1 == suffix2:
|
||||||
|
return False
|
||||||
|
|
||||||
|
domain1 = url1.host()[:-len(suffix1)].split('.')[-1]
|
||||||
|
domain2 = url2.host()[:-len(suffix2)].split('.')[-1]
|
||||||
|
return domain1 == domain2
|
||||||
|
|
||||||
|
|
||||||
class FuzzyUrlError(Exception):
|
class FuzzyUrlError(Exception):
|
||||||
|
|
||||||
"""Exception raised by fuzzy_url on problems.
|
"""Exception raised by fuzzy_url on problems.
|
||||||
|
@ -497,3 +497,21 @@ def test_fuzzy_url_error(url, raising, has_err_string):
|
|||||||
else:
|
else:
|
||||||
expected_text = "Error message"
|
expected_text = "Error message"
|
||||||
assert str(excinfo.value) == expected_text
|
assert str(excinfo.value) == expected_text
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('are_same, url1, url2', [
|
||||||
|
(True, 'http://example.com', 'http://www.example.com'),
|
||||||
|
(True, 'http://bbc.co.uk', 'https://www.bbc.co.uk'),
|
||||||
|
(True, 'http://many.levels.of.domains.example.com', 'http://www.example.com'),
|
||||||
|
(True, 'http://idn.иком.museum', 'http://idn2.иком.museum'),
|
||||||
|
(True, 'http://one.not_a_valid_tld', 'http://one.not_a_valid_tld'),
|
||||||
|
|
||||||
|
(False, 'http://bbc.co.uk', 'http://example.co.uk'),
|
||||||
|
(False, 'https://example.kids.museum', 'http://example.kunst.museum'),
|
||||||
|
(False, 'http://idn.иком.museum', 'http://idn.ירושלים.museum'),
|
||||||
|
(False, 'http://one.not_a_valid_tld', 'http://two.not_a_valid_tld'),
|
||||||
|
])
|
||||||
|
def test_same_domain(are_same, url1, url2):
|
||||||
|
"""Tests for same_domain."""
|
||||||
|
assert urlutils.same_domain(QUrl(url1), QUrl(url2)) == are_same
|
||||||
|
assert urlutils.same_domain(QUrl(url2), QUrl(url1)) == are_same
|
||||||
|
Loading…
Reference in New Issue
Block a user