diff --git a/doc/qutebrowser.1.asciidoc b/doc/qutebrowser.1.asciidoc
index 2a403de5f..4c8ce2556 100644
--- a/doc/qutebrowser.1.asciidoc
+++ b/doc/qutebrowser.1.asciidoc
@@ -99,6 +99,9 @@ show it.
 *--temp-basedir*::
     Use a temporary basedir.
 
+*--no-err-windows*::
+    Don't show any error windows (used for tests/smoke.py).
+
 *--qt-name* 'NAME'::
     Set the window name.
 
diff --git a/qutebrowser/app.py b/qutebrowser/app.py
index 02f8af907..e75e9e4e9 100644
--- a/qutebrowser/app.py
+++ b/qutebrowser/app.py
@@ -29,7 +29,7 @@ import time
 import shutil
 import tempfile
 
-from PyQt5.QtWidgets import QApplication, QMessageBox
+from PyQt5.QtWidgets import QApplication
 from PyQt5.QtGui import QDesktopServices, QPixmap, QIcon, QCursor, QWindow
 from PyQt5.QtCore import (pyqtSlot, qInstallMessageHandler, QTimer, QUrl,
                           QObject, Qt, QEvent)
@@ -49,7 +49,7 @@ from qutebrowser.mainwindow import mainwindow
 from qutebrowser.misc import readline, ipc, savemanager, sessions, crashsignal
 from qutebrowser.misc import utilcmds  # pylint: disable=unused-import
 from qutebrowser.utils import (log, version, message, utils, qtutils, urlutils,
-                               objreg, usertypes, standarddir)
+                               objreg, usertypes, standarddir, error)
 # We import utilcmds to run the cmdutils.register decorators.
 
 
@@ -91,7 +91,7 @@ def run(args):
     try:
         sent = ipc.send_to_running_instance(args)
         if sent:
-            sys.exit(0)
+            sys.exit(usertypes.Exit.ok)
         log.init.debug("Starting IPC server...")
         server = ipc.IPCServer(args, qApp)
         objreg.register('ipc-server', server)
@@ -103,14 +103,14 @@ def run(args):
         time.sleep(500)
         sent = ipc.send_to_running_instance(args)
         if sent:
-            sys.exit(0)
+            sys.exit(usertypes.Exit.ok)
         else:
-            ipc.display_error(e)
-            sys.exit(1)
+            ipc.display_error(e, args)
+            sys.exit(usertypes.Exit.err_ipc)
     except ipc.Error as e:
-        ipc.display_error(e)
+        ipc.display_error(e, args)
         # We didn't really initialize much so far, so we just quit hard.
-        sys.exit(1)
+        sys.exit(usertypes.Exit.err_ipc)
 
     init(args, crash_handler)
     ret = qt_mainloop()
@@ -144,11 +144,9 @@ def init(args, crash_handler):
     try:
         _init_modules(args, crash_handler)
     except (OSError, UnicodeDecodeError) as e:
-        msgbox = QMessageBox(
-            QMessageBox.Critical, "Error while initializing!",
-            "Error while initializing: {}".format(e))
-        msgbox.exec_()
-        sys.exit(1)
+        error.handle_fatal_exc(e, args, "Error while initializing!",
+                               pre_text="Error while initializing")
+        sys.exit(usertypes.Exit.err_init)
     QTimer.singleShot(0, functools.partial(_process_args, args))
 
     log.init.debug("Initializing eventfilter...")
@@ -398,7 +396,8 @@ def _init_modules(args, crash_handler):
     log.init.debug("Initializing web history...")
     history.init(qApp)
     log.init.debug("Initializing crashlog...")
-    crash_handler.handle_segfault()
+    if not args.no_err_windows:
+        crash_handler.handle_segfault()
     log.init.debug("Initializing sessions...")
     sessions.init(qApp)
     log.init.debug("Initializing js-bridge...")
@@ -636,10 +635,9 @@ class Quitter:
                 try:
                     save_manager.save(key, is_exit=True)
                 except OSError as e:
-                    msgbox = QMessageBox(
-                        QMessageBox.Critical, "Error while saving!",
-                        "Error while saving {}: {}".format(key, e))
-                    msgbox.exec_()
+                    error.handle_fatal_exc(
+                        e, self._args, "Error while saving!",
+                        pre_text="Error while saving {}".format(key))
         # Re-enable faulthandler to stdout, then remove crash log
         log.destroy.debug("Deactivating crash log...")
         objreg.get('crash-handler').destroy_crashlogfile()
diff --git a/qutebrowser/config/config.py b/qutebrowser/config/config.py
index af5cbaf08..e195310ef 100644
--- a/qutebrowser/config/config.py
+++ b/qutebrowser/config/config.py
@@ -33,12 +33,12 @@ import collections
 import collections.abc
 
 from PyQt5.QtCore import pyqtSignal, pyqtSlot, QObject, QUrl, QSettings
-from PyQt5.QtWidgets import QMessageBox
 
 from qutebrowser.config import configdata, configexc, textwrapper
 from qutebrowser.config.parsers import ini, keyconf
 from qutebrowser.commands import cmdexc, cmdutils
-from qutebrowser.utils import message, objreg, utils, standarddir, log, qtutils
+from qutebrowser.utils import (message, objreg, utils, standarddir, log,
+                               qtutils, error, usertypes)
 from qutebrowser.utils.usertypes import Completion
 
 
@@ -137,8 +137,8 @@ def _init_main_config(parent=None):
     Args:
         parent: The parent to pass to ConfigManager.
     """
+    args = objreg.get('args')
     try:
-        args = objreg.get('args')
         config_obj = ConfigManager(standarddir.config(), 'qutebrowser.conf',
                                    args.relaxed_config, parent=parent)
     except (configexc.Error, configparser.Error, UnicodeDecodeError) as e:
@@ -149,12 +149,11 @@ def _init_main_config(parent=None):
                 e.section, e.option)  # pylint: disable=no-member
         except AttributeError:
             pass
-        errstr += "\n{}".format(e)
-        msgbox = QMessageBox(QMessageBox.Critical,
-                             "Error while reading config!", errstr)
-        msgbox.exec_()
+        errstr += "\n"
+        error.handle_fatal_exc(e, args, "Error while reading config!",
+                               pre_text=errstr)
         # We didn't really initialize much so far, so we just quit hard.
-        sys.exit(1)
+        sys.exit(usertypes.Exit.err_config)
     else:
         objreg.register('config', config_obj)
         if standarddir.config() is not None:
@@ -178,8 +177,8 @@ def _init_key_config(parent):
     Args:
         parent: The parent to use for the KeyConfigParser.
     """
+    args = objreg.get('args')
     try:
-        args = objreg.get('args')
         key_config = keyconf.KeyConfigParser(standarddir.config(), 'keys.conf',
                                              args.relaxed_config,
                                              parent=parent)
@@ -188,12 +187,10 @@ def _init_key_config(parent):
         errstr = "Error while reading key config:\n"
         if e.lineno is not None:
             errstr += "In line {}: ".format(e.lineno)
-        errstr += str(e)
-        msgbox = QMessageBox(QMessageBox.Critical,
-                             "Error while reading key config!", errstr)
-        msgbox.exec_()
+        error.handle_fatal_exc(e, args, "Error while reading key config!",
+                               pre_text=errstr)
         # We didn't really initialize much so far, so we just quit hard.
-        sys.exit(1)
+        sys.exit(usertypes.Exit.err_key_config)
     else:
         objreg.register('key-config', key_config)
         if standarddir.config() is not None:
diff --git a/qutebrowser/misc/checkpyver.py b/qutebrowser/misc/checkpyver.py
index 5bb41dd1a..cdf1ebbe3 100644
--- a/qutebrowser/misc/checkpyver.py
+++ b/qutebrowser/misc/checkpyver.py
@@ -47,7 +47,7 @@ def check_python_version():
         version_str = '.'.join(map(str, sys.version_info[:3]))
         text = ("At least Python 3.4 is required to run qutebrowser, but " +
                 version_str + " is installed!\n")
-        if Tk:
+        if Tk and '--no-err-windows' not in sys.argv:
             root = Tk()
             root.withdraw()
             messagebox.showerror("qutebrowser: Fatal error!", text)
diff --git a/qutebrowser/misc/crashdialog.py b/qutebrowser/misc/crashdialog.py
index bc00bce06..79a425fa9 100644
--- a/qutebrowser/misc/crashdialog.py
+++ b/qutebrowser/misc/crashdialog.py
@@ -584,3 +584,38 @@ class ReportErrorDialog(QDialog):
         btn.clicked.connect(self.close)
         hbox.addWidget(btn)
         vbox.addLayout(hbox)
+
+
+def dump_exception_info(exc, pages, cmdhist, objects):
+    """Dump exception info to stderr.
+
+    Args:
+        exc: An exception tuple (type, value, traceback)
+        pages: A list of lists of the open pages (URLs as strings)
+        cmdhist: A list with the command history (as strings)
+        objects: A list of all QObjects as string.
+    """
+    print(file=sys.stderr)
+    print("\n\n===== Handling exception with --no-err-windows... =====\n\n",
+          file=sys.stderr)
+    print("\n---- Exceptions ----", file=sys.stderr)
+    print(''.join(traceback.format_exception(*exc)), file=sys.stderr)
+    print("\n---- Version info ----", file=sys.stderr)
+    try:
+        print(version.version(), file=sys.stderr)
+    except Exception:
+        traceback.print_exc()
+    print("\n---- Config ----", file=sys.stderr)
+    try:
+        conf = objreg.get('config')
+        print(conf.dump_userconfig(), file=sys.stderr)
+    except Exception:
+        traceback.print_exc()
+    print("\n---- Commandline args ----", file=sys.stderr)
+    print(' '.join(sys.argv[1:]), file=sys.stderr)
+    print("\n---- Open pages ----", file=sys.stderr)
+    print('\n\n'.join('\n'.join(e) for e in pages), file=sys.stderr)
+    print("\n---- Command history ----", file=sys.stderr)
+    print('\n'.join(cmdhist), file=sys.stderr)
+    print("\n---- Objects ----", file=sys.stderr)
+    print(objects, file=sys.stderr)
diff --git a/qutebrowser/misc/crashsignal.py b/qutebrowser/misc/crashsignal.py
index 48d14a797..c860cdb6e 100644
--- a/qutebrowser/misc/crashsignal.py
+++ b/qutebrowser/misc/crashsignal.py
@@ -121,6 +121,7 @@ class CrashHandler(QObject):
 
     def _init_crashlogfile(self):
         """Start a new logfile and redirect faulthandler to it."""
+        assert not self._args.no_err_windows
         data_dir = standarddir.data()
         if data_dir is None:
             return
@@ -224,18 +225,21 @@ class CrashHandler(QObject):
         except TypeError:
             log.destroy.exception("Error while preventing shutdown")
         self._app.closeAllWindows()
-        self._crash_dialog = crashdialog.ExceptionCrashDialog(
-            self._args.debug, pages, cmd_history, exc, objects)
-        ret = self._crash_dialog.exec_()
-        if ret == QDialog.Accepted:  # restore
-            self._quitter.restart(pages)
+        if self._args.no_err_windows:
+            crashdialog.dump_exception_info(exc, pages, cmd_history, objects)
+        else:
+            self._crash_dialog = crashdialog.ExceptionCrashDialog(
+                self._args.debug, pages, cmd_history, exc, objects)
+            ret = self._crash_dialog.exec_()
+            if ret == QDialog.Accepted:  # restore
+                self._quitter.restart(pages)
 
         # We might risk a segfault here, but that's better than continuing to
         # run in some undefined state, so we only do the most needed shutdown
         # here.
         qInstallMessageHandler(None)
         self.destroy_crashlogfile()
-        sys.exit(1)
+        sys.exit(usertypes.Exit.exception)
 
     def raise_crashdlg(self):
         """Raise the crash dialog if one exists."""
diff --git a/qutebrowser/misc/earlyinit.py b/qutebrowser/misc/earlyinit.py
index 2307cc8c0..0416ed9a1 100644
--- a/qutebrowser/misc/earlyinit.py
+++ b/qutebrowser/misc/earlyinit.py
@@ -84,12 +84,15 @@ def _die(message, exception=None):
         print(file=sys.stderr)
         traceback.print_exc()
     app = QApplication(sys.argv)
-    message += '<br/><br/><br/><b>Error:</b><br/>{}'.format(exception)
-    msgbox = QMessageBox(QMessageBox.Critical, "qutebrowser: Fatal error!",
-                         message)
-    msgbox.setTextFormat(Qt.RichText)
-    msgbox.resize(msgbox.sizeHint())
-    msgbox.exec_()
+    if '--no-err-windows' in sys.argv:
+        print("Exiting because of --no-err-windows.", file=sys.stderr)
+    else:
+        message += '<br/><br/><br/><b>Error:</b><br/>{}'.format(exception)
+        msgbox = QMessageBox(QMessageBox.Critical, "qutebrowser: Fatal error!",
+                             message)
+        msgbox.setTextFormat(Qt.RichText)
+        msgbox.resize(msgbox.sizeHint())
+        msgbox.exec_()
     app.quit()
     sys.exit(1)
 
@@ -186,13 +189,13 @@ def check_pyqt_core():
         text = text.replace('</b>', '')
         text = text.replace('<br />', '\n')
         text += '\n\nError: {}'.format(e)
-        if tkinter:
+        if tkinter and '--no-err-windows' not in sys.argv:
             root = tkinter.Tk()
             root.withdraw()
             tkinter.messagebox.showerror("qutebrowser: Fatal error!", text)
         else:
             print(text, file=sys.stderr)
-        if '--debug' in sys.argv:
+        if '--debug' in sys.argv or '--no-err-windows' in sys.argv:
             print(file=sys.stderr)
             traceback.print_exc()
         sys.exit(1)
diff --git a/qutebrowser/misc/ipc.py b/qutebrowser/misc/ipc.py
index ba0ec87c9..749eb4e4e 100644
--- a/qutebrowser/misc/ipc.py
+++ b/qutebrowser/misc/ipc.py
@@ -27,9 +27,8 @@ import hashlib
 
 from PyQt5.QtCore import pyqtSignal, pyqtSlot, QObject
 from PyQt5.QtNetwork import QLocalSocket, QLocalServer, QAbstractSocket
-from PyQt5.QtWidgets import QMessageBox
 
-from qutebrowser.utils import log, usertypes
+from qutebrowser.utils import log, usertypes, error
 
 
 CONNECT_TIMEOUT = 100
@@ -130,12 +129,12 @@ class IPCServer(QObject):
                 self._socketname))
 
     @pyqtSlot(int)
-    def on_error(self, error):
+    def on_error(self, err):
         """Convenience method which calls _socket_error on an error."""
         self._timer.stop()
         log.ipc.debug("Socket error {}: {}".format(
             self._socket.error(), self._socket.errorString()))
-        if error != QLocalSocket.PeerClosedError:
+        if err != QLocalSocket.PeerClosedError:
             _socket_error("handling IPC connection", self._socket)
 
     @pyqtSlot()
@@ -282,9 +281,8 @@ def send_to_running_instance(args):
             return False
 
 
-def display_error(exc):
+def display_error(exc, args):
     """Display a message box with an IPC error."""
-    text = '{}\n\nMaybe another instance is running but frozen?'.format(exc)
-    msgbox = QMessageBox(QMessageBox.Critical, "Error while connecting to "
-                         "running instance!", text)
-    msgbox.exec_()
+    error.handle_fatal_exc(
+        exc, args, "Error while connecting to running instance!",
+        post_text="Maybe another instance is running but frozen?")
diff --git a/qutebrowser/qutebrowser.py b/qutebrowser/qutebrowser.py
index 03f3c582b..b2d61fb48 100644
--- a/qutebrowser/qutebrowser.py
+++ b/qutebrowser/qutebrowser.py
@@ -96,6 +96,8 @@ def get_argparser():
                        help="Drop into pdb on exceptions.")
     debug.add_argument('--temp-basedir', action='store_true', help="Use a "
                        "temporary basedir.")
+    debug.add_argument('--no-err-windows', action='store_true', help="Don't "
+                       "show any error windows (used for tests/smoke.py).")
     # For the Qt args, we use store_const with const=True rather than
     # store_true because we want the default to be None, to make
     # utils.qt:get_args easier.
diff --git a/qutebrowser/utils/error.py b/qutebrowser/utils/error.py
new file mode 100644
index 000000000..d637f4d33
--- /dev/null
+++ b/qutebrowser/utils/error.py
@@ -0,0 +1,54 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Tools related to error printing/displaying."""
+
+from PyQt5.QtWidgets import QMessageBox
+
+from qutebrowser.utils import log
+
+
+def handle_fatal_exc(exc, args, title, *, pre_text='', post_text=''):
+    """Handle a fatal "expected" exception by displaying an error box.
+
+    If --no-err-windows is given as argument, the text is logged to the error
+    logger instead.
+
+    Args:
+        exc: The Exception object being handled.
+        args: The argparser namespace.
+        title: The title to be used for the error message.
+        pre_text: The text to be displayed before the exception text.
+        post_text: The text to be displayed after the exception text.
+    """
+    if args.no_err_windows:
+        log.misc.exception("Handling fatal {} with --no-err-windows!".format(
+            exc.__class__.__name__))
+        log.misc.error("title: {}".format(title))
+        log.misc.error("pre_text: {}".format(pre_text))
+        log.misc.error("post_text: {}".format(post_text))
+    else:
+        if pre_text:
+            msg_text = '{}: {}'.format(pre_text, exc)
+        else:
+            msg_text = str(exc)
+        if post_text:
+            msg_text += '\n\n{}'.format(post_text)
+        msgbox = QMessageBox(QMessageBox.Critical, title, msg_text)
+        msgbox.exec_()
diff --git a/qutebrowser/utils/usertypes.py b/qutebrowser/utils/usertypes.py
index 7371f3114..28a30f941 100644
--- a/qutebrowser/utils/usertypes.py
+++ b/qutebrowser/utils/usertypes.py
@@ -240,6 +240,11 @@ Completion = enum('Completion', ['command', 'section', 'option', 'value',
                                  'quickmark_by_name', 'url', 'sessions'])
 
 
+# Exit statuses for errors
+Exit = enum('Exit', ['ok', 'reserved', 'exception', 'err_ipc', 'err_init',
+                     'err_config', 'err_key_config'])
+
+
 class Question(QObject):
 
     """A question asked to the user, e.g. via the status bar.
diff --git a/tox.ini b/tox.ini
index e2bdf35f7..28b0c4bf9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -112,6 +112,13 @@ commands =
     git --no-pager diff --exit-code --stat
     {envpython} scripts/asciidoc2html.py {posargs}
 
+[testenv:smoke]
+deps =
+    -rrequirements.txt
+commands =
+    {[testenv:mkvenv]commands}
+    {envpython} -m qutebrowser --debug --no-err-windows --nowindow --temp-basedir about:blank ":later 500 quit"
+
 [pytest]
 norecursedirs = .tox .venv
 markers =