diff --git a/scripts/dev/check_coverage.py b/scripts/dev/check_coverage.py
index 3217e4dfd..89208b037 100644
--- a/scripts/dev/check_coverage.py
+++ b/scripts/dev/check_coverage.py
@@ -34,6 +34,8 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
 
 from scripts import utils
 
+import qutebrowser
+
 
 Message = collections.namedtuple('Message', 'typ, text')
 MsgType = enum.Enum('MsgType', 'insufficent_coverage, perfect_file')
@@ -169,6 +171,14 @@ def check(fileobj, perfect_files):
 
     for klass in classes:
         filename = klass.attrib['filename']
+
+        basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..',
+                                               '..'))
+        if os.path.isabs(filename):
+            common_path = os.path.commonpath([basedir, filename])
+            if common_path:
+                filename = filename[len(common_path):].lstrip('/')
+
         line_cov = float(klass.attrib['line-rate']) * 100
         branch_cov = float(klass.attrib['branch-rate']) * 100
 
diff --git a/tests/unit/scripts/test_check_coverage.py b/tests/unit/scripts/test_check_coverage.py
index 054ed58df..a458132a2 100644
--- a/tests/unit/scripts/test_check_coverage.py
+++ b/tests/unit/scripts/test_check_coverage.py
@@ -51,7 +51,8 @@ class CovtestHelper:
         coveragerc = str(self._testdir.tmpdir / 'coveragerc')
         return self._testdir.runpytest('--cov=module',
                                        '--cov-config={}'.format(coveragerc),
-                                       '--cov-report=xml')
+                                       '--cov-report=xml',
+                                       plugins=['no:faulthandler'])
 
     def check(self, perfect_files=None):
         """Run check_coverage.py and run its return value."""
diff --git a/tox.ini b/tox.ini
index e650e7082..9f17d2c11 100644
--- a/tox.ini
+++ b/tox.ini
@@ -26,7 +26,7 @@ deps =
     parse==1.6.6
     parse-type==0.3.4
     py==1.4.30
-    pytest==2.7.3  # rq.filter: <2.8.0
+    pytest==2.8.2
     pytest-bdd==2.15.0
     pytest-catchlog==1.2.0
     pytest-cov==2.2.0