From 774bcbf6b3b588f3a601d003bd0b9f348c2b1d3e Mon Sep 17 00:00:00 2001
From: Florian Bruhin <git@the-compiler.org>
Date: Thu, 4 Feb 2016 06:43:14 +0100
Subject: [PATCH] tests: Don't fail on teardown too if test failed.

When a end-to-end test failed which would've marked an error message as
expected later in the test, seeing the teardown message about an unexpected
error being logged is really confusing.
---
 tests/conftest.py                     | 11 +++++++++++
 tests/integration/quteprocess.py      | 16 +++++++++++++---
 tests/integration/test_quteprocess.py | 15 ++++++++++++---
 3 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 6b9c1feca..14db6c44d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -435,3 +435,14 @@ def pytest_configure(config):
 def pytest_unconfigure(config):
     if config.xvfb_display is not None:
         config.xvfb_display.stop()
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    """Make test information available in fixtures.
+
+    See http://pytest.org/latest/example/simple.html#making-test-result-information-available-in-fixtures
+    """
+    outcome = yield
+    rep = outcome.get_result()
+    setattr(item, "rep_" + rep.when, rep)
diff --git a/tests/integration/quteprocess.py b/tests/integration/quteprocess.py
index 1670e3190..661473f33 100644
--- a/tests/integration/quteprocess.py
+++ b/tests/integration/quteprocess.py
@@ -259,11 +259,21 @@ class QuteProc(testprocess.Process):
         if skip_texts:
             pytest.skip(', '.join(skip_texts))
 
-    def after_test(self):
+    def after_test(self, did_fail):
+        """Handle unexpected/skip logging and clean up after each test.
+
+        Args:
+            did_fail: Set if the main test failed already, then logged errors
+                      are ignored.
+        """
         __tracebackhide__ = True
         bad_msgs = [msg for msg in self._data
                     if self._is_error_logline(msg) and not msg.expected]
 
+        if did_fail:
+            super().after_test()
+            return
+
         try:
             if bad_msgs:
                 text = 'Logged unexpected errors:\n\n' + '\n'.join(
@@ -399,7 +409,7 @@ def quteproc(quteproc_process, httpbin, request):
     request.node._quteproc_log = quteproc_process.captured_log
     quteproc_process.before_test()
     yield quteproc_process
-    quteproc_process.after_test()
+    quteproc_process.after_test(did_fail=request.node.rep_call.failed)
 
 
 @pytest.yield_fixture
@@ -410,4 +420,4 @@ def quteproc_new(qapp, httpbin, request):
     request.node._quteproc_log = proc.captured_log
     # Not calling before_test here as that would start the process
     yield proc
-    proc.after_test()
+    proc.after_test(did_fail=request.node.rep_call.failed)
diff --git a/tests/integration/test_quteprocess.py b/tests/integration/test_quteprocess.py
index 03c20b32f..872ec73de 100644
--- a/tests/integration/test_quteprocess.py
+++ b/tests/integration/test_quteprocess.py
@@ -40,7 +40,16 @@ def test_quteproc_error_message(qtbot, quteproc, cmd):
     # Usually we wouldn't call this from inside a test, but here we force the
     # error to occur during the test rather than at teardown time.
     with pytest.raises(pytest.fail.Exception):
-        quteproc.after_test()
+        quteproc.after_test(did_fail=False)
+
+
+def test_quteproc_error_message_did_fail(qtbot, quteproc):
+    """Make sure the test does not fail on teardown if the main test failed."""
+    with qtbot.waitSignal(quteproc.got_error):
+        quteproc.send_cmd(':message-error test')
+    # Usually we wouldn't call this from inside a test, but here we force the
+    # error to occur during the test rather than at teardown time.
+    quteproc.after_test(did_fail=True)
 
 
 def test_quteproc_skip_via_js(qtbot, quteproc):
@@ -50,7 +59,7 @@ def test_quteproc_skip_via_js(qtbot, quteproc):
 
         # Usually we wouldn't call this from inside a test, but here we force
         # the error to occur during the test rather than at teardown time.
-        quteproc.after_test()
+        quteproc.after_test(did_fail=False)
 
     assert str(excinfo.value) == 'test'
 
@@ -74,7 +83,7 @@ def test_quteprocess_quitting(qtbot, quteproc_process):
     with qtbot.waitSignal(quteproc_process.proc.finished, timeout=15000):
         quteproc_process.send_cmd(':quit')
     with pytest.raises(testprocess.ProcessExited):
-        quteproc_process.after_test()
+        quteproc_process.after_test(did_fail=False)
 
 
 @pytest.mark.parametrize('data, attrs', [