diff --git a/qutebrowser/completion/completiondelegate.py b/qutebrowser/completion/completiondelegate.py
index 779906a83..076b99bed 100644
--- a/qutebrowser/completion/completiondelegate.py
+++ b/qutebrowser/completion/completiondelegate.py
@@ -203,8 +203,9 @@ class CompletionItemDelegate(QStyledItemDelegate):
             columns_to_filter = index.model().columns_to_filter(index)
             if index.column() in columns_to_filter and pattern:
                 repl = r'<span class="highlight">\g<0></span>'
-                pat = html.escape(re.escape(pattern)).replace(r'\ ', r'|')
-                txt = html.escape(self._opt.text)
+                pat = html.escape(re.escape(pattern), quote=False).replace(
+                    r'\ ', r'|')
+                txt = html.escape(self._opt.text, quote=False)
                 text = re.sub(pat, repl, txt, flags=re.IGNORECASE)
                 self._doc.setHtml(text)
             else:
diff --git a/tests/unit/completion/test_completiondelegate.py b/tests/unit/completion/test_completiondelegate.py
index b90c096a2..9d22002ee 100644
--- a/tests/unit/completion/test_completiondelegate.py
+++ b/tests/unit/completion/test_completiondelegate.py
@@ -86,7 +86,10 @@ def delegate(mock_style_option, mock_text_document, config_stub, mocker, view):
     ('a b', 'cadb', 'c{a}d{b}'),
     ('foo', '<foo>', '&lt;{foo}&gt;'),
     ('<a>', "<a>bc", '{&lt;a&gt;}bc'),
-    ('foo', "'foo'", "&#x27;{foo}&#x27;"),
+
+    # https://github.com/qutebrowser/qutebrowser/issues/4199
+    ('foo', "'foo'", "'{foo}'"),
+    ('x', "'x'", "'{x}'"),
 ])
 def test_paint(delegate, painter, view, mock_style_option, mock_text_document,
                pat, txt_in, txt_out):