diff --git a/scripts/dev/run_pylint_on_tests.py b/scripts/dev/run_pylint_on_tests.py
index 6c426561d..3e2d3d0b5 100644
--- a/scripts/dev/run_pylint_on_tests.py
+++ b/scripts/dev/run_pylint_on_tests.py
@@ -56,6 +56,8 @@ def main():
         'protected-access',
         # https://bitbucket.org/logilab/pylint/issue/511/
         'undefined-variable',
+        # directories without __init__.py...
+        'import-error',
     ]
 
     toxinidir = sys.argv[1]
diff --git a/tests/conftest.py b/tests/conftest.py
index 78d83be17..9fd73fe73 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-# pylint: disable=unused-import,import-error
+# pylint: disable=unused-import
 
 """The qutebrowser test suite conftest file."""
 
diff --git a/tests/helpers/test_logfail.py b/tests/helpers/test_logfail.py
index daccb49eb..9ade5a4f9 100644
--- a/tests/helpers/test_logfail.py
+++ b/tests/helpers/test_logfail.py
@@ -23,7 +23,7 @@
 import logging
 
 import pytest
-import pytest_catchlog  # pylint: disable=import-error
+import pytest_catchlog
 
 
 def test_log_debug():
diff --git a/tests/integration/testprocess.py b/tests/integration/testprocess.py
index 6d6f8f277..c641aa9a1 100644
--- a/tests/integration/testprocess.py
+++ b/tests/integration/testprocess.py
@@ -23,7 +23,7 @@ import re
 import os
 import time
 
-import pytestqt.plugin  # pylint: disable=import-error
+import pytestqt.plugin
 from PyQt5.QtCore import pyqtSlot, pyqtSignal, QProcess, QObject, QElapsedTimer
 from PyQt5.QtTest import QSignalSpy