rnhmjoj/qutebrowser
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Replace some tests with a generic host blocking list

Browse Source
This commit is contained in:
Corentin Julé 2015-12-07 00:09:51 +01:00
parent a24a7790cd
commit 6be0ff67f7
1 changed files with 204 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

317
tests/unit/browser/test_adblock.py
View File

@ -32,23 +32,25 @@ from qutebrowser.browser import adblock
from qutebrowser.utils import objreg from qutebrowser.utils import objreg
from qutebrowser.commands import cmdexc from qutebrowser.commands import cmdexc
WHITELISTED_HOSTS = ('qutebrowser.org', 'goodhost.gov')
WHITELISTED_HOSTS = ['qutebrowser.org', BLOCKED_HOSTS = ('verygoodhost.com',
'mediumsite.org'] 'goodhost.gov',
'mediumhost.io',
BLOCKED_HOSTS = ['localhost', 'malware.badhost.org',
'mediumsite.io', '4-verybadhost.com',
'badsite.org', 'ads.worsthostever.net',
'verybadsite.com', 'localhost')
'worstsiteever.net']
URLS_TO_CHECK = ['http://verybadsite.com',
'http://badsite.org',
'http://localhost',
'http://qutebrowser.org',
'http://testa.com',
'http://testb.com']
URLS_TO_CHECK = ('http://localhost',
'ftp://goodhost.gov',
'http://mediumhost.io',
'http://malware.badhost.org',
'http://4-verybadhost.com',
'http://ads.worsthostever.net',
'http://verygoodhost.com',
'ftp://perfecthost.com',
'http://qutebrowser.org')
@pytest.fixture @pytest.fixture
def data_tmpdir(monkeypatch, tmpdir): def data_tmpdir(monkeypatch, tmpdir):
@ -110,21 +112,29 @@ def download_stub(win_registry):
def create_zipfile(files, directory): def create_zipfile(files, directory, zipname='test'):
"""Returns a zipfile populated with given files and its name.""" """Returns a zipfile populated with given files and its name."""
directory = str(directory) directory = str(directory)
zipfile_path = os.path.join(directory, 'test.zip') zipfile_path = os.path.join(directory, zipname + '.zip')
with zipfile.ZipFile(zipfile_path, 'w') as new_zipfile: with zipfile.ZipFile(zipfile_path, 'w') as new_zipfile:
for file_name in files: for file_name in files:
new_zipfile.write(file_name, arcname=os.path.basename(file_name)) new_zipfile.write(file_name, arcname=os.path.basename(file_name))
# Removes path from file name # Removes path from file name
return new_zipfile, zipfile_path return new_zipfile, zipfile_path
def create_blocklist(directory, blocked_hosts=BLOCKED_HOSTS, name='hosts'): def create_blocklist(directory, blocked_hosts=BLOCKED_HOSTS,
name='hosts', line_format=None):
"""Returns a QUrl instance linking to a file with given name in given """Returns a QUrl instance linking to a file with given name in given
directory which contains a list of given blocked_hosts.""" directory which contains a list of given blocked_hosts."""
blocklist = QUrl(os.path.join(str(directory), name)) blocklist = QUrl(os.path.join(str(directory), name))
with open(blocklist.path(), 'w', encoding='UTF-8') as hosts: with open(blocklist.path(), 'w', encoding='UTF-8') as hosts:
hosts.write('# Blocked Hosts List #\n\n')
if line_format == 'etc_hosts':
for path in blocked_hosts:
hosts.write('127.0.0.1 ' + path + '\n')
elif line_format == 'no_hosts':
hosts.write('This file is not a hosts file')
else: #one host per line format
for path in blocked_hosts: for path in blocked_hosts:
hosts.write(path + '\n') hosts.write(path + '\n')
return blocklist return blocklist
@ -136,7 +146,7 @@ def assert_urls(host_blocker,
"""Test if urls_to_check are effectively blocked or not by HostBlocker """Test if urls_to_check are effectively blocked or not by HostBlocker
Url in blocked_hosts and not in whitelisted_hosts should be blocked Url in blocked_hosts and not in whitelisted_hosts should be blocked
All other Urls should not be blocked.""" All other Urls should not be blocked."""
whitelisted_hosts += list(host_blocker.WHITELISTED) whitelisted_hosts = list(whitelisted_hosts) + list(host_blocker.WHITELISTED)
for str_url in urls_to_check: for str_url in urls_to_check:
url = QUrl(str_url) url = QUrl(str_url)
host = url.host() host = url.host()
@ -150,14 +160,34 @@ class TestHostBlockerUpdate:
"""Tests for function adblock_update of class HostBlocker.""" """Tests for function adblock_update of class HostBlocker."""
def generic_blocklists(self, directory):
file1 = create_blocklist(directory, BLOCKED_HOSTS[5:], 'hosts', 'etc_hosts')
file2 = create_blocklist(directory, name='README', line_format='no_hosts')
file3 = create_blocklist(directory, BLOCKED_HOSTS[:2], 'false_positive')
files_to_zip = [file1.path(), file2.path(), file3.path()]
zip_path = create_zipfile(files_to_zip, directory, 'block1')[1]
remote_blocklist1 = QUrl(zip_path)
blocklist2 = create_blocklist(directory, [BLOCKED_HOSTS[4]], 'malwarelist', 'etc_hosts')
zip_path = create_zipfile([blocklist2.path()], directory, 'block2')[1]
remote_blocklist2 = QUrl(zip_path)
# A local list with one host per line
local_blocklist3 = create_blocklist(directory, [BLOCKED_HOSTS[3]], 'mycustomblocklist', 'etc_hosts')
local_blocklist3.setScheme('file')
# A list that cannot be read due to its formatting
remote_blocklist4 = create_blocklist(directory, [BLOCKED_HOSTS[2]], 'badlist', 'no_hosts')
return [remote_blocklist1, remote_blocklist2, local_blocklist3, remote_blocklist4]
def test_without_datadir(self, config_stub, tmpdir, def test_without_datadir(self, config_stub, tmpdir,
monkeypatch, win_registry): monkeypatch, win_registry):
"""No directory for data configured so no hosts file exists. """No directory for data configured so no hosts file exists.
CommandError is raised by adblock_update CommandError is raised by adblock_update
Ensure no url is blocked.""" Ensure no url is blocked."""
blocklist = create_blocklist(tmpdir)
config_stub.data = {'content': config_stub.data = {'content':
{'host-block-lists': [blocklist], {'host-block-lists': self.generic_blocklists(tmpdir),
'host-blocking-enabled': True}} 'host-blocking-enabled': True}}
monkeypatch.setattr('qutebrowser.utils.standarddir.data', monkeypatch.setattr('qutebrowser.utils.standarddir.data',
lambda: None) lambda: None)
@ -171,9 +201,8 @@ class TestHostBlockerUpdate:
def test_disabled_blocking(self, basedir, config_stub, download_stub, def test_disabled_blocking(self, basedir, config_stub, download_stub,
data_tmpdir, tmpdir, win_registry): data_tmpdir, tmpdir, win_registry):
"""Ensure that no url is blocked when host blocking is disabled.""" """Ensure that no url is blocked when host blocking is disabled."""
blocklist = create_blocklist(tmpdir)
config_stub.data = {'content': config_stub.data = {'content':
{'host-block-lists': [blocklist], {'host-block-lists': self.generic_blocklists(tmpdir),
'host-blocking-enabled': False}} 'host-blocking-enabled': False}}
host_blocker = adblock.HostBlocker() host_blocker = adblock.HostBlocker()
host_blocker.adblock_update(0) host_blocker.adblock_update(0)
@ -194,118 +223,170 @@ class TestHostBlockerUpdate:
for str_url in URLS_TO_CHECK: for str_url in URLS_TO_CHECK:
assert not host_blocker.is_blocked(QUrl(str_url)) assert not host_blocker.is_blocked(QUrl(str_url))
def test_remote_text(self, config_stub, basedir, download_stub, def test_successful_update(self, config_stub, basedir, download_stub,
data_tmpdir, tmpdir, win_registry): data_tmpdir, tmpdir, win_registry):
"""Update with single fakely remote text blocklist.
Ensure urls from hosts in this blocklist get blocked."""
blocklist = create_blocklist(tmpdir)
config_stub.data = {'content': config_stub.data = {'content':
{'host-block-lists': [blocklist], {'host-block-lists': self.generic_blocklists(tmpdir),
'host-blocking-enabled': True, 'host-blocking-enabled': True,
'host-blocking-whitelist': None}} 'host-blocking-whitelist': None}}
host_blocker = adblock.HostBlocker() host_blocker = adblock.HostBlocker()
host_blocker.adblock_update(0) host_blocker.adblock_update(0)
# Simulate download is finished # Simulate download is finished
# XXX Is it ok to use private attribute hostblocker._in_progress ? # XXX Is it ok to use private attribute hostblocker._in_progress ?
while host_blocker._in_progress != []:
host_blocker._in_progress[0].finished.emit() host_blocker._in_progress[0].finished.emit()
host_blocker.read_hosts() host_blocker.read_hosts()
assert_urls(host_blocker, whitelisted_hosts=[]) assert_urls(host_blocker, BLOCKED_HOSTS[3:], whitelisted_hosts=[])
def test_remote_zip_single(self, config_stub, basedir, download_stub, # def test_remote_text(self, config_stub, basedir, download_stub,
data_tmpdir, tmpdir, win_registry): # data_tmpdir, tmpdir, win_registry):
"""Update with single fakely remote zip containing one blocklist file. # """Update with single fakely remote text blocklist.
Ensure urls from hosts in this blocklist get blocked.""" # Ensure urls from hosts in this blocklist get blocked."""
blocklist = create_blocklist(tmpdir) # blocklist = create_blocklist(tmpdir)
zip_url = QUrl(create_zipfile([blocklist.path()], tmpdir)[1]) # config_stub.data = {'content':
config_stub.data = {'content': # {'host-block-lists': [blocklist],
{'host-block-lists': [zip_url], # 'host-blocking-enabled': True,
'host-blocking-enabled': True, # 'host-blocking-whitelist': None}}
'host-blocking-whitelist': None}} # host_blocker = adblock.HostBlocker()
host_blocker = adblock.HostBlocker() # host_blocker.adblock_update(0)
host_blocker.adblock_update(0) # # Simulate download is finished
host_blocker._in_progress[0].finished.emit() # # XXX Is it ok to use private attribute hostblocker._in_progress ?
host_blocker.read_hosts() # host_blocker._in_progress[0].finished.emit()
assert_urls(host_blocker, whitelisted_hosts=[]) # host_blocker.read_hosts()
# assert_urls(host_blocker, whitelisted_hosts=[])
# FIXME adblock.guess_zip_filename should raise FileNotFound Error # def test_remote_zip_single(self, config_stub, basedir, download_stub,
# as no files in the zip are called hosts # data_tmpdir, tmpdir, win_registry):
def test_remote_zip_multi1(self, config_stub, basedir, download_stub, # """Update with single fakely remote zip containing one blocklist file.
data_tmpdir, tmpdir, win_registry): # Ensure urls from hosts in this blocklist get blocked."""
"""Update with single fakely remote zip containing two files. # blocklist = create_blocklist(tmpdir)
None of them is called hosts, FileNotFoundError should be raised.""" # zip_url = QUrl(create_zipfile([blocklist.path()], tmpdir)[1])
file1 = create_blocklist(tmpdir, name='file1.txt') # config_stub.data = {'content':
file2_hosts = ['testa.com', 'testb.com'] # {'host-block-lists': [zip_url],
file2 = create_blocklist(tmpdir, file2_hosts, name='file2.txt') # 'host-blocking-enabled': True,
files_to_zip = [file1.path(), file2.path()] # 'host-blocking-whitelist': None}}
zip_path = create_zipfile(files_to_zip, tmpdir)[1] # host_blocker = adblock.HostBlocker()
zip_url = QUrl(zip_path) # host_blocker.adblock_update(0)
config_stub.data = {'content': # host_blocker._in_progress[0].finished.emit()
{'host-block-lists': [zip_url], # host_blocker.read_hosts()
'host-blocking-enabled': True, # assert_urls(host_blocker, whitelisted_hosts=[])
'host-blocking-whitelist': None}}
host_blocker = adblock.HostBlocker()
host_blocker.adblock_update(0)
host_blocker._in_progress[0].finished.emit()
host_blocker.read_hosts()
#with pytest.raises(FileNotFoundError):
for str_url in URLS_TO_CHECK:
assert not host_blocker.is_blocked(QUrl(str_url))
def test_remote_zip_multi2(self, config_stub, basedir, download_stub, # # FIXME adblock.guess_zip_filename should raise FileNotFound Error
data_tmpdir, tmpdir, win_registry): # # as no files in the zip are called hosts
"""Update with single fakely remote zip containing two files. # def test_remote_zip_multi1(self, config_stub, basedir, download_stub,
One of them is called hosts and should be used as blocklist. # data_tmpdir, tmpdir, win_registry):
Ensure urls from hosts in this blocklist get blocked # """Update with single fakely remote zip containing two files.
and the hosts from the other file are not.""" # None of them is called hosts, FileNotFoundError should be raised."""
file1 = create_blocklist(tmpdir, name='hosts.txt') # file1 = create_blocklist(tmpdir, name='file1.txt')
file2_hosts = ['testa.com', 'testb.com'] # file2_hosts = ['testa.com', 'testb.com']
file2 = create_blocklist(tmpdir, file2_hosts, name='file2.txt') # file2 = create_blocklist(tmpdir, file2_hosts, name='file2.txt')
files_to_zip = [file1.path(), file2.path()] # files_to_zip = [file1.path(), file2.path()]
zip_path = create_zipfile(files_to_zip, tmpdir)[1] # zip_path = create_zipfile(files_to_zip, tmpdir)[1]
zip_url = QUrl(zip_path) # zip_url = QUrl(zip_path)
config_stub.data = {'content': # config_stub.data = {'content':
{'host-block-lists': [zip_url], # {'host-block-lists': [zip_url],
'host-blocking-enabled': True, # 'host-blocking-enabled': True,
'host-blocking-whitelist': None}} # 'host-blocking-whitelist': None}}
host_blocker = adblock.HostBlocker() # host_blocker = adblock.HostBlocker()
host_blocker.adblock_update(0) # #with pytest.raises(FileNotFoundError):
host_blocker._in_progress[0].finished.emit() # host_blocker.adblock_update(0)
host_blocker.read_hosts() # host_blocker._in_progress[0].finished.emit()
assert_urls(host_blocker, whitelisted_hosts=[]) # host_blocker.read_hosts()
# for str_url in URLS_TO_CHECK:
# assert not host_blocker.is_blocked(QUrl(str_url))
def test_local_text(self, config_stub, basedir, download_stub, # def test_remote_zip_multi2(self, config_stub, basedir, download_stub,
# data_tmpdir, tmpdir, win_registry):
# """Update with single fakely remote zip containing two files.
# One of them is called hosts and should be used as blocklist.
# Ensure urls from hosts in this blocklist get blocked
# and the hosts from the other file are not."""
# file1 = create_blocklist(tmpdir, name='hosts.txt')
# file2_hosts = ['testa.com', 'testb.com']
# file2 = create_blocklist(tmpdir, file2_hosts, name='file2.txt')
# files_to_zip = [file1.path(), file2.path()]
# zip_path = create_zipfile(files_to_zip, tmpdir)[1]
# zip_url = QUrl(zip_path)
# config_stub.data = {'content':
# {'host-block-lists': [zip_url],
# 'host-blocking-enabled': True,
# 'host-blocking-whitelist': None}}
# host_blocker = adblock.HostBlocker()
# host_blocker.adblock_update(0)
# host_blocker._in_progress[0].finished.emit()
# host_blocker.read_hosts()
# assert_urls(host_blocker, whitelisted_hosts=[])
# def test_local_text(self, config_stub, basedir, download_stub,
# data_tmpdir, tmpdir, win_registry):
# """Update with single local text blocklist.
# Ensure urls from hosts in this blocklist get blocked."""
# blocklist = create_blocklist(tmpdir)
# blocklist.setScheme("file")
# config_stub.data = {'content':
# {'host-block-lists': [blocklist],
# 'host-blocking-enabled': True,
# 'host-blocking-whitelist': None}}
# host_blocker = adblock.HostBlocker()
# host_blocker.adblock_update(0)
# host_blocker.read_hosts()
# assert_urls(host_blocker, whitelisted_hosts=[])
# def test_local_zip_single(self, config_stub, basedir, download_stub,
# data_tmpdir, tmpdir, win_registry):
# """Update with single local zip containing one file.
# Ensure urls from hosts in this blocklist get blocked."""
# blocklist = create_blocklist(tmpdir)
# zip_url = QUrl(create_zipfile([blocklist.path()], tmpdir)[1])
# zip_url.setScheme('file')
# config_stub.data = {'content':
# {'host-block-lists': [zip_url],
# 'host-blocking-enabled': True,
# 'host-blocking-whitelist': None}}
# host_blocker = adblock.HostBlocker()
# host_blocker.adblock_update(0)
# host_blocker.read_hosts()
# assert_urls(host_blocker, whitelisted_hosts=[])
# def test_local_zip_multi(self, config_stub, basedir, download_stub,
# data_tmpdir, tmpdir, win_registry):
# """Update with single local zip containing two files.
# One of them is called hosts and should be used as blocklist.
# Ensure urls from hosts in this blocklist get blocked
# and the hosts from the other file are not."""
# file1 = create_blocklist(tmpdir, name='hosts.txt')
# file2_hosts = ['testa.com', 'testb.com']
# file2 = create_blocklist(tmpdir, file2_hosts, name='file2.txt')
# files_to_zip = [file1.path(), file2.path()]
# zip_path = create_zipfile(files_to_zip, tmpdir)[1]
# zip_url = QUrl(zip_path)
# zip_url.setScheme('file')
# config_stub.data = {'content':
# {'host-block-lists': [zip_url],
# 'host-blocking-enabled': True,
# 'host-blocking-whitelist': None}}
# host_blocker = adblock.HostBlocker()
# host_blocker.adblock_update(0)
# host_blocker.read_hosts()
# assert_urls(host_blocker, whitelisted_hosts=[])
class TestHostBlockerIsBlocked:
"""Tests for function adblock_update of class HostBlocker."""
def test_with_whitelist(self, config_stub, basedir, download_stub,
data_tmpdir, tmpdir, win_registry): data_tmpdir, tmpdir, win_registry):
"""Update with single local text blocklist. """"""
Ensure urls from hosts in this blocklist get blocked.""" # Simulate adblock_update has already been run,
blocklist = create_blocklist(tmpdir) # exclude localhost from blocked-hosts file
blocklist.setScheme("file") filtered_blocked_hosts = BLOCKED_HOSTS[:-1]
blocklist = create_blocklist(tmpdir,
blocked_hosts=filtered_blocked_hosts,
name="blocked-hosts")
config_stub.data = {'content': config_stub.data = {'content':
{'host-block-lists': [blocklist], {'host-block-lists': [blocklist],
'host-blocking-enabled': True, 'host-blocking-enabled': True,
'host-blocking-whitelist': None}} 'host-blocking-whitelist': WHITELISTED_HOSTS}}
host_blocker = adblock.HostBlocker() host_blocker = adblock.HostBlocker()
host_blocker.adblock_update(0)
host_blocker.read_hosts() host_blocker.read_hosts()
assert_urls(host_blocker, whitelisted_hosts=[]) assert_urls(host_blocker)
def test_local_zip_multi(self, config_stub, basedir, download_stub,
data_tmpdir, tmpdir, win_registry):
"""Update with single local zip containing two files.
One of them is called hosts and should be used as blocklist.
Ensure urls from hosts in this blocklist get blocked
and the hosts from the other file are not."""
file1 = create_blocklist(tmpdir, name='hosts.txt')
file2_hosts = ['testa.com', 'testb.com']
file2 = create_blocklist(tmpdir, file2_hosts, name='file2.txt')
files_to_zip = [file1.path(), file2.path()]
zip_path = create_zipfile(files_to_zip, tmpdir)[1]
zip_url = QUrl(zip_path)
zip_url.setScheme('file')
config_stub.data = {'content':
{'host-block-lists': [zip_url],
'host-blocking-enabled': True,
'host-blocking-whitelist': None}}
host_blocker = adblock.HostBlocker()
host_blocker.adblock_update(0)
host_blocker.read_hosts()
assert_urls(host_blocker, whitelisted_hosts=[])