diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py
index cc3d1817e..ed2028d86 100644
--- a/tests/utils/test_utils.py
+++ b/tests/utils/test_utils.py
@@ -98,9 +98,12 @@ class TestReadFile:
 
     @pytest.fixture(autouse=True, params=[True, False])
     def freezer(self, request, monkeypatch):
-        if request.param:
+        if request.param and not getattr(sys, 'frozen', False):
             monkeypatch.setattr(sys, 'frozen', True, raising=False)
             monkeypatch.setattr('sys.executable', qutebrowser.__file__)
+        elif not request.param and getattr(sys, 'frozen', False):
+            # Want to test unfrozen tests, but we are frozen
+            pytest.skip("Can't run with sys.frozen = True!")
 
     def test_readfile(self):
         """Read a test file."""