diff --git a/tests/helpers/stubs.py b/tests/helpers/stubs.py
index 711b1105f..64bc793cb 100644
--- a/tests/helpers/stubs.py
+++ b/tests/helpers/stubs.py
@@ -261,6 +261,9 @@ class FakeWebTab(browsertab.AbstractTab):
     def load_status(self):
         return self._load_status
 
+    def shutdown(self):
+        pass
+
 
 class FakeSignal:
 
diff --git a/tests/unit/mainwindow/test_tabwidget.py b/tests/unit/mainwindow/test_tabwidget.py
index 1f0ff55d5..7ad22fcc3 100644
--- a/tests/unit/mainwindow/test_tabwidget.py
+++ b/tests/unit/mainwindow/test_tabwidget.py
@@ -23,7 +23,7 @@ import pytest
 
 from PyQt5.QtGui import QIcon, QPixmap
 
-from qutebrowser.mainwindow import tabwidget
+from qutebrowser.mainwindow import tabwidget, tabbedbrowser
 from qutebrowser.utils import usertypes
 
 
@@ -39,6 +39,14 @@ class TestTabWidget:
                             usertypes.Backend.QtWebKit)
         return w
 
+    @pytest.fixture
+    def browser(self, qtbot, monkeypatch, config_stub):
+        w = tabbedbrowser.TabbedBrowser(win_id=0, private=False)
+        qtbot.addWidget(w)
+        monkeypatch.setattr(tabwidget.objects, 'backend',
+                            usertypes.Backend.QtWebKit)
+        return w
+
     def test_small_icon_doesnt_crash(self, widget, qtbot, fake_web_tab):
         """Test that setting a small icon doesn't produce a crash.
 
@@ -53,15 +61,29 @@ class TestTabWidget:
         with qtbot.waitExposed(widget):
             widget.show()
 
+    @pytest.mark.parametrize("num_tabs", [4, 10])
     def test_update_tab_titles_benchmark(self, benchmark, widget,
-                                         qtbot, fake_web_tab):
+                                         qtbot, fake_web_tab, num_tabs):
         """Benchmark for update_tab_titles."""
-        widget.addTab(fake_web_tab(), 'foobar')
-        widget.addTab(fake_web_tab(), 'foobar2')
-        widget.addTab(fake_web_tab(), 'foobar3')
-        widget.addTab(fake_web_tab(), 'foobar4')
+        for i in range(num_tabs):
+            widget.addTab(fake_web_tab(), 'foobar' + str(i))
 
         with qtbot.waitExposed(widget):
             widget.show()
 
         benchmark(widget._update_tab_titles)
+
+    @pytest.mark.parametrize("num_tabs", [4, 10])
+    def test_add_remove_tab_benchmark(self, benchmark, browser,
+                                      qtbot, fake_web_tab, num_tabs):
+        """Benchmark for addTab and removeTab."""
+        def _run_bench():
+            for i in range(num_tabs):
+                browser.addTab(fake_web_tab(), 'foobar' + str(i))
+
+            with qtbot.waitExposed(browser):
+                browser.show()
+
+            browser.shutdown()
+
+        benchmark(_run_bench)