diff --git a/.appveyor.yml b/.appveyor.yml
index 456603a39..abaf0de72 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -12,6 +12,7 @@ install:
 
 test_script:
   - C:\Python34\Scripts\tox -e py34
+  - C:\Python34\Scripts\tox -e py34-integration
   - C:\Python34\Scripts\tox -e unittests-frozen
   - C:\Python34\Scripts\tox -e smoke-frozen
   - C:\Python34\Scripts\tox -e pylint
diff --git a/.travis.yml b/.travis.yml
index a1e4ed51f..8012257a0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ install:
   - python scripts/dev/ci_install.py
 
 script:
-    - xvfb-run -s "-screen 0 640x480x16" tox -e py34
+    - xvfb-run -s "-screen 0 640x480x16" tox -e py34,py34-integration
     - tox -e misc
     - tox -e pep257
     - tox -e pyflakes
diff --git a/tests/conftest.py b/tests/conftest.py
index f7164e1b9..bed1dd107 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -66,6 +66,15 @@ def pytest_collection_modifyitems(items):
                     True, reason="No DISPLAY available")
                 item.add_marker(skip_marker)
 
+        if hasattr(item, 'module'):
+            module_path = os.path.relpath(
+                item.module.__file__,
+                os.path.commonprefix([__file__, item.module.__file__]))
+
+            module_root_dir = os.path.split(module_path)[0]
+            if module_root_dir == 'integration':
+                item.add_marker(pytest.mark.integration)
+
 
 def pytest_runtest_setup(item):
     """Add some custom markers."""
diff --git a/tests/integration/test_smoke.py b/tests/integration/test_smoke.py
new file mode 100644
index 000000000..b85ccce4d
--- /dev/null
+++ b/tests/integration/test_smoke.py
@@ -0,0 +1,33 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test which simply runs qutebrowser to check if it starts properly."""
+
+
+import sys
+import subprocess
+
+import pytest
+
+
+@pytest.mark.not_frozen
+def test_smoke():
+    argv = [sys.executable, '-m', 'qutebrowser', '--debug', '--no-err-windows',
+            '--nowindow', '--temp-basedir', 'about:blank', ':later 500 quit']
+    subprocess.check_call(argv)
diff --git a/tox.ini b/tox.ini
index b20ec5f7f..fefc11ca5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py34,misc,pep257,pyflakes,pep8,mccabe,pylint,pyroma,check-manifest
+envlist = py34,py34-integration,misc,pep257,pyflakes,pep8,mccabe,pylint,pyroma,check-manifest
 
 [testenv]
 # https://bitbucket.org/hpk42/tox/issue/246/ - only needed for Windows though
@@ -27,9 +27,16 @@ deps =
     beautifulsoup4==4.4.0
 commands =
     {envpython} scripts/link_pyqt.py --tox {envdir}
-    {envpython} -m py.test --strict -rfEsw --cov qutebrowser --cov-report xml --cov-report=html --cov-report= {posargs:tests}
+    {envpython} -m py.test --strict -rfEsw -m 'not integration' --cov qutebrowser --cov-report xml --cov-report=html --cov-report= {posargs:tests}
     {envpython} scripts/dev/check_coverage.py {posargs}
-    {envpython} -m qutebrowser --no-err-windows --nowindow --temp-basedir about:blank ":later 500 quit"
+
+[testenv:py34-integration]
+setenv = {[testenv]setenv}
+passenv = {[testenv]passenv}
+deps = {[testenv]deps}
+commands =
+    {envpython} scripts/link_pyqt.py --tox {envdir}
+    {envpython} -m py.test --strict -rfEsw -m 'integration' {posargs:tests}
 
 [testenv:mkvenv]
 basepython = python3