diff --git a/tests/conftest.py b/tests/conftest.py
index 26ad52d37..d016f2e58 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -35,6 +35,7 @@ from helpers import logfail
 from helpers.logfail import fail_on_logging
 from helpers.messagemock import message_mock
 from helpers.fixtures import *  # pylint: disable=wildcard-import
+from qutebrowser.utils import qtutils
 
 
 # Set hypothesis settings
@@ -131,8 +132,7 @@ def pytest_collection_modifyitems(config, items):
                 reason='JS prompts are not supported with this PyQt version'))
         if item.get_marker('issue2183'):
             item.add_marker(pytest.mark.xfail(
-                bool(os.environ.get('QUTE_BDD_WEBENGINE', None) and
-                     os.environ.get('DOCKER', None)),
+                config.webengine and qtutils.version_check('5.7.1'),
                 reason='https://github.com/The-Compiler/qutebrowser/issues/'
                        '2183'))