diff --git a/.appveyor.yml b/.appveyor.yml
index abaf0de72..2a01ba3dc 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -14,5 +14,4 @@ test_script:
   - C:\Python34\Scripts\tox -e py34
   - C:\Python34\Scripts\tox -e py34-integration
   - C:\Python34\Scripts\tox -e unittests-frozen
-  - C:\Python34\Scripts\tox -e smoke-frozen
   - C:\Python34\Scripts\tox -e pylint
diff --git a/scripts/dev/freeze.py b/scripts/dev/freeze.py
index f3fecd757..b04d951e7 100755
--- a/scripts/dev/freeze.py
+++ b/scripts/dev/freeze.py
@@ -101,26 +101,27 @@ bdist_mac_options = {
     'bundle_name': 'qutebrowser',
 }
 
-if sys.platform.startswith('win'):
-    base = 'Win32GUI'
-    target_name = 'qutebrowser.exe'
-else:
-    base = None
-    target_name = 'qutebrowser'
 
-executable = cx.Executable('qutebrowser/__main__.py', base=base,
-                           targetName=target_name,
-                           shortcutName='qutebrowser',
-                           shortcutDir='ProgramMenuFolder',
-                           icon=os.path.join(BASEDIR, 'icons',
-                                             'qutebrowser.ico'))
+def get_exe(base, target_name):
+    return cx.Executable('qutebrowser/__main__.py', base=base,
+                         targetName=target_name, shortcutName='qutebrowser',
+                         shortcutDir='ProgramMenuFolder',
+                         icon=os.path.join(BASEDIR, 'icons',
+                                           'qutebrowser.ico'))
 
 
 if __name__ == '__main__':
+    if sys.platform.startswith('win'):
+        base = 'Win32GUI'
+        target_name = 'qutebrowser.exe'
+    else:
+        base = None
+        target_name = 'qutebrowser'
+
     try:
         setupcommon.write_git_file()
         cx.setup(
-            executables=[executable],
+            executables=[get_exe(base, target_name)],
             options={
                 'build_exe': get_build_exe_options(),
                 'bdist_msi': bdist_msi_options,
diff --git a/scripts/dev/freeze_tests.py b/scripts/dev/freeze_tests.py
index eae600e71..24c5a5211 100755
--- a/scripts/dev/freeze_tests.py
+++ b/scripts/dev/freeze_tests.py
@@ -69,12 +69,14 @@ def get_build_exe_options():
 
 def main():
     """Main entry point."""
+    base = 'Win32GUI' if sys.platform.startswith('win') else None
     with temp_git_commit_file():
         cx.setup(
             executables=[cx.Executable('scripts/dev/run_frozen_tests.py',
                                        targetName='run-frozen-tests'),
                          cx.Executable('tests/integration/webserver_sub.py',
-                                       targetName='webserver_sub')],
+                                       targetName='webserver_sub'),
+                         freeze.get_exe(base, target_name='qutebrowser')],
             options={'build_exe': get_build_exe_options()},
             **setupcommon.setupdata
         )
diff --git a/tests/integration/test_smoke.py b/tests/integration/test_smoke.py
index b85ccce4d..d7eea2b97 100644
--- a/tests/integration/test_smoke.py
+++ b/tests/integration/test_smoke.py
@@ -21,13 +21,17 @@
 
 
 import sys
+import os.path
 import subprocess
 
 import pytest
 
 
-@pytest.mark.not_frozen
 def test_smoke():
-    argv = [sys.executable, '-m', 'qutebrowser', '--debug', '--no-err-windows',
-            '--nowindow', '--temp-basedir', 'about:blank', ':later 500 quit']
+    if hasattr(sys, 'frozen'):
+        argv = [os.path.join(os.path.dirname(sys.executable), 'qutebrowser')]
+    else:
+        argv = [sys.executable, '-m', 'qutebrowser']
+    argv += ['--debug', '--no-err-windows', '--nowindow', '--temp-basedir',
+             'about:blank', ':later 500 quit']
     subprocess.check_call(argv)
diff --git a/tox.ini b/tox.ini
index 54445add9..e6d74a599 100644
--- a/tox.ini
+++ b/tox.ini
@@ -176,18 +176,6 @@ commands =
     git --no-pager diff --exit-code --stat
     {envpython} scripts/asciidoc2html.py {posargs}
 
-[testenv:smoke-frozen]
-basepython = python3
-passenv = {[testenv]passenv}
-skip_install = true
-deps =
-    {[testenv]deps}
-    cx_Freeze==4.3.4
-commands =
-    {envpython} scripts/link_pyqt.py --tox {envdir}
-    {envpython} scripts/dev/freeze.py build_exe --qute-skip-html -b {envdir}/build
-    {envdir}/build/qutebrowser --no-err-windows --nowindow --temp-basedir about:blank ":later 500 quit"
-
 [testenv:cxfreeze-windows]
 # PYTHON is actually required when using this env, but the entire tox.ini would
 # fail if we didn't have a fallback defined.