diff --git a/scripts/dev/freeze_tests.py b/scripts/dev/freeze_tests.py
index a84d0c9f9..e5160483f 100755
--- a/scripts/dev/freeze_tests.py
+++ b/scripts/dev/freeze_tests.py
@@ -30,6 +30,8 @@ import cx_Freeze as cx  # pylint: disable=import-error
 # cx_Freeze is hard to install (needs C extensions) so we don't check for it.
 import pytest
 
+import httpbin
+
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir,
                 os.pardir))
 from scripts import setupcommon
@@ -52,7 +54,15 @@ def get_build_exe_options():
     """Get build_exe options with additional includes."""
     opts = freeze.get_build_exe_options(skip_html=True)
     opts['includes'] += pytest.freeze_includes()  # pylint: disable=no-member
-    opts['includes'] += ['unittest.mock', 'PyQt5.QtTest', 'hypothesis', 'bs4']
+    opts['includes'] += ['unittest.mock', 'PyQt5.QtTest', 'hypothesis', 'bs4',
+                         'httpbin', 'jinja2.ext']
+
+    httpbin_dir = os.path.dirname(httpbin.__file__)
+    opts['include_files'] += [
+        ('tests/integration/data', 'integration/data'),
+        (os.path.join(httpbin_dir, 'templates'), 'integration/templates'),
+    ]
+
     opts['packages'].append('qutebrowser')
     return opts
 
@@ -62,7 +72,9 @@ def main():
     with temp_git_commit_file():
         cx.setup(
             executables=[cx.Executable('scripts/dev/run_frozen_tests.py',
-                                       targetName='run-frozen-tests')],
+                                       targetName='run-frozen-tests'),
+                         cx.Executable('tests/integration/webserver.py',
+                                       targetName='webserver')],
             options={'build_exe': get_build_exe_options()},
             **setupcommon.setupdata
         )
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 75389604d..b8a33ea6a 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -125,8 +125,15 @@ class HTTPBin(QObject):
 
     def start(self):
         """Start the webserver."""
-        filename = os.path.join(os.path.dirname(__file__), 'webserver.py')
-        self.proc.start(sys.executable, [filename, str(self.port)])
+        if hasattr(sys, 'frozen'):
+            executable = os.path.join(os.path.dirname(sys.executable),
+                                      'webserver')
+            args = []
+        else:
+            executable = sys.executable
+            args = [os.path.join(os.path.dirname(__file__), 'webserver.py')]
+
+        self.proc.start(executable, args + [str(self.port)])
         ok = self.proc.waitForStarted()
         assert ok
         self.proc.readyRead.connect(self.read_log)
diff --git a/tests/integration/webserver.py b/tests/integration/webserver.py
index 1456643a2..c2ef4cb05 100644
--- a/tests/integration/webserver.py
+++ b/tests/integration/webserver.py
@@ -31,9 +31,22 @@ import flask
 
 @app.route('/data/<path:path>')
 def send_data(path):
-    basedir = os.path.realpath(os.path.dirname(__file__))
+    if hasattr(sys, 'frozen'):
+        basedir = os.path.realpath(os.path.dirname(sys.executable))
+        data_dir = os.path.join(basedir, 'integration', 'data')
+    else:
+        basedir = os.path.realpath(os.path.dirname(__file__))
+        data_dir = os.path.join(basedir, 'data')
     print(basedir)
-    return flask.send_from_directory(os.path.join(basedir, 'data'), path)
+    return flask.send_from_directory(data_dir, path)
 
 
-app.run(port=int(sys.argv[1]), debug=True, use_reloader=False)
+def main():
+    if hasattr(sys, 'frozen'):
+        basedir = os.path.realpath(os.path.dirname(sys.executable))
+        app.template_folder = os.path.join(basedir, 'integration', 'templates')
+    app.run(port=int(sys.argv[1]), debug=True, use_reloader=False)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tox.ini b/tox.ini
index cc784f61a..54445add9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -56,7 +56,7 @@ basepython = python3
 passenv = {[testenv]passenv}
 skip_install = true
 deps =
-    {[testenv]deps}
+    {[testenv:py34-integration]deps}
     cx_Freeze==4.3.4
 commands =
     {envpython} scripts/link_pyqt.py --tox {envdir}