tests: Add an SSL server subprocess.

tests/integration/conftest.py

@@ -21,6 +21,6 @@
 
 """Things needed for integration testing."""
 
-from webserver import httpbin, httpbin_after_test
+from webserver import httpbin, httpbin_after_test, ssl_server
 from quteprocess import quteproc_process, quteproc
 from testprocess import pytest_runtest_makereport

tests/integration/data/ssl/cert.csr

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICpTCCAY0CAQAwYDElMCMGA1UECgwccXV0ZWJyb3dzZXIgdGVzdCBjZXJ0aWZp
+Y2F0ZTESMBAGA1UEAwwJbG9jYWxob3N0MSMwIQYJKoZIhvcNAQkBFhRtYWlsQHF1
+dGVicm93c2VyLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAO77
+e6QqjeGDjq8tDCGSEi+7m/cDL6PbX8zNNKoVplcoJjoPC/6KmdsLin4SO3iAd5ti
+XOpPQqyCBgBUd7axP5Ya6M6rhWJaYUczUMdx8bRr4mdaTbd/UhVM/dI1vS/LvBKH
+OY+8k3E6Neb5jeDe2dfXgokURL4c/jIS1MDumvYCAteoHRYvjGcTSDERr0DT0DY4
+oPyrImabSHRGXLz0euQsMY4d9ZTakomYH52cRMNEOKArU1ARNZ0UyHzumuSkjIFV
+G5PFgMra0tgAPdCA1sx51cQUBOYxnqMdgOBThonrbusYYR17D7TqsvC6R9E0HWhF
+b4JJkPB3EDVEzWqQFgcCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4IBAQBC7JrJuHyF
+YFiujBlXFZIQrPNW7FF28zqBuXLfviwVBF/sKmNMKwC0nUgmCb/wFPxv3yrj+7az
+r29FWSGVhs6k15GVsqSwnbSJDznh/W1elWwpTo2GODMmRY3VeYSY9WiQUhe5KA5x
+56p5Kgtl53wZzdl+Pi93xVYAZFWl2O3GFs4f+GCrORjHC7ejZoq6xfRzNLZbLF0a
+QyptcnYaZSppDB/nZx4p75GKcj9qWXaJbT8mjqJdgRCFPyUkQjSY6WEEAP3LXrXx
+ThZUekv81Jh+kPTZjSd1d24Bd0nFkQdFf8SRn21jnP+PrzipBOdvm+bT8dI/71xg
+8ZJ631jogV4L
+-----END CERTIFICATE REQUEST-----

tests/integration/data/ssl/cert.pem

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDPDCCAiQCCQCHskwLQC4vHDANBgkqhkiG9w0BAQsFADBgMSUwIwYDVQQKDBxx
+dXRlYnJvd3NlciB0ZXN0IGNlcnRpZmljYXRlMRIwEAYDVQQDDAlsb2NhbGhvc3Qx
+IzAhBgkqhkiG9w0BCQEWFG1haWxAcXV0ZWJyb3dzZXIub3JnMB4XDTE2MDExMjE4
+NDYyM1oXDTI2MDEwOTE4NDYyM1owYDElMCMGA1UECgwccXV0ZWJyb3dzZXIgdGVz
+dCBjZXJ0aWZpY2F0ZTESMBAGA1UEAwwJbG9jYWxob3N0MSMwIQYJKoZIhvcNAQkB
+FhRtYWlsQHF1dGVicm93c2VyLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAO77e6QqjeGDjq8tDCGSEi+7m/cDL6PbX8zNNKoVplcoJjoPC/6KmdsL
+in4SO3iAd5tiXOpPQqyCBgBUd7axP5Ya6M6rhWJaYUczUMdx8bRr4mdaTbd/UhVM
+/dI1vS/LvBKHOY+8k3E6Neb5jeDe2dfXgokURL4c/jIS1MDumvYCAteoHRYvjGcT
+SDERr0DT0DY4oPyrImabSHRGXLz0euQsMY4d9ZTakomYH52cRMNEOKArU1ARNZ0U
+yHzumuSkjIFVG5PFgMra0tgAPdCA1sx51cQUBOYxnqMdgOBThonrbusYYR17D7Tq
+svC6R9E0HWhFb4JJkPB3EDVEzWqQFgcCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA
+lTuJK8wseifpepUaWIev+59ulxxMzeippi+xqoYnjrNjINNdk5Wh+Dj7Crb5R8dn
+afkC+XE9PMKEvKBmQZj/KVEL/G7bjZBA73oibKpBMWIdxaIwSFN2Xq4zKWLHESrb
+2Wy8MiehZiSdgUtnmTPM0BlDmc6u9/0nLdCjsBoKYVOLw2FDcD1P8NOJT0dUjSUu
+aYmUakcn+lQEjuBplrsGvL0vCGR/kzG2vwoTuGnx66HURuHU6E7yBTQ2diyhzOQc
+sMwwDfrsY19K3IH6AuVcCgGit1LE/zCqMFQuFrIhYB5Mt5bLSeWVBDzKClxZB0Di
+OxK2sWZvLdGLsFltKB+IJA==
+-----END CERTIFICATE-----

tests/integration/data/ssl/key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA7vt7pCqN4YOOry0MIZISL7ub9wMvo9tfzM00qhWmVygmOg8L
+/oqZ2wuKfhI7eIB3m2Jc6k9CrIIGAFR3trE/lhrozquFYlphRzNQx3HxtGviZ1pN
+t39SFUz90jW9L8u8Eoc5j7yTcTo15vmN4N7Z19eCiRREvhz+MhLUwO6a9gIC16gd
+Fi+MZxNIMRGvQNPQNjig/KsiZptIdEZcvPR65Cwxjh31lNqSiZgfnZxEw0Q4oCtT
+UBE1nRTIfO6a5KSMgVUbk8WAytrS2AA90IDWzHnVxBQE5jGeox2A4FOGietu6xhh
+HXsPtOqy8LpH0TQdaEVvgkmQ8HcQNUTNapAWBwIDAQABAoIBADysrryEbVdHLm+9
+USooyuNBj5yMO4kvhkgaBXf1XTEdqW7uKQ5sJBnf+T5+5Ih4nWVe+NYoX3Yq4Nku
+mOJSaCF1HYxzMb9B0RbhqW2puUMkbOvumnKvKajszjiTmj/LSymtGWkr6IdDzzGg
+RGxGSCqrtaGV+soF1GfkLg35xnAUnwk3pfVqGyXl66+bCCWcqXZTUlOB55KEa+5F
+9rkMlS6/X3DGZLvON7ZtZqZe7E8Foo9qU1VSHHfxIkS5P4UNxjf7woQogmhNTRT6
+tX0SmDQdP59sdFJ09Expr2AfSFxfkGuQf+JSG/JMprg0ub0ksw7UZvaW1uJNKL9I
+XQSVPgECgYEA94DlPsGd8wWllMjOIEDkERUP2s4uJjPb6jodqewf9tuyxuwRnpOs
+fb5uq7mMJXG3sszqom0q3DBoapNdCX1vTywWHKc1Nik5PT7jbEXFaRLfvA/F8WfF
+6Rugm/S+nezTc7XhtDnOpfl+7wFSJy0we0C3RvxJqAaLaQRDobeNiQcCgYEA9y+z
+wdXaOcJnC5bPO3ollFewX00WJaAAFpDnfqC3ALJx94/xJVJW6A7TZnKKJmWQ/bFz
+0iuyhMe3Nd2yzAhl0qs0lmVe2V2tgJO/CVVP8OQmwlHKSZssDCjaBrHIkNwdL00j
+qtSYg/FafLPL24AFSr25+sBn/FfxHTzlWVlWywECgYAUyjX3dIoQ/NtwyQFPgkPm
+D2/agFEuElMZtLIDMPtqX///Z5r/SAZINbPUJuzXxFqa4U2gQS1Fe6d5tFEvV+L+
+soRU+dKlbwcI1vyBfsbbUaOLh4OoCIB+WTy/fOp6F4eXg6Km4egy1udLqj+9XLVi
+1QfQJacGPy58rsgDkIiKBwKBgHtVtd91kNlZAolpyiTnIXEO/9XNZMuJNgIMczVf
+g3A5mVvo2m3A09Qd8aUgaYYXD21F6YBohT5zWBrsb5YWapffDPItylGyyCtrjNpf
+Uu/jJuO2Y7SuVCANEhxdALIm4fkECFPol+DdwESQgZsYGYvddrqC3l+ukYQBKn6W
+cRQBAoGBAMA8tN67zOtZWalkokLHPDDK/TRUI/+Idc7xX7Rx/KZLuhfT26pLbe5Q
+onbhe+TSq+4aYfUdcWJE2oM8DQn6CrNZFXKhz/0DLE+leASwwJLNCBbDdLjij2sy
+7x2VeGKVG7V2KEhqcDUH/TO0e9PeGnz0vnebzN2+EZue6J9OTfLr
+-----END RSA PRIVATE KEY-----

tests/integration/data/ssl/privkey.pem

@@ -0,0 +1,30 @@
+-----BEGIN ENCRYPTED PRIVATE KEY-----
+MIIFDjBABgkqhkiG9w0BBQ0wMzAbBgkqhkiG9w0BBQwwDgQISAypJ52ykvkCAggA
+MBQGCCqGSIb3DQMHBAgcPyy/O0hXkgSCBMgA4rZIvVKE73SsGCpJou1LgGAuPX1m
+qyOPwRGC9T1p1HPaMcucIKpZPp5JSx3B9xwN/V+gpi3XXU1oTLaJhXwOpp8v106l
+lR9Us91o4nUWVmo2C6nG1z/GSP573RBqjxChiHQchjT5UufKOi+/0elg6tgpu2cv
+k+CLcgKp80dUr+UOPLAqIC2B+ex4BQHPrki+wbsTeMoZaXnPcTbl0OjABXbG6X4l
+Gf2xftM7I+Wr/E7dnOEHGwUUH4hAzqflgUTHTZZtUDU7v99ggBBRux5vp9Zi2gWp
+ksuAmxfPDMcE1Mpu+ZTZ3+cp4TWuWwKRpCX9USjmwnkhdEhqc/arHID/Db+SNP6z
+lrdHY7BeAWcwDTo+4KZAEK7LKpAukRvpLcyvufo/smGaXsYytFz6Un8scSoySuqo
+TEKyAioxNsOGJ2Xz5Jt+tdNLO/5W4jCuvwPx1GDlumwPcMHjDrXlZUa0qfoJCcun
+lptbxZfqd7ouXLy1OF5FAsLs/iCmBwsyOS/qysFwq442WEwT/qn3ZoGBNkkJahu4
+OQ5sA14+nZHsBp1+iXZZxKmAERvQfFIRY0oe+Hmdwvyzb4mbIgFyPzU0CRFb+L1/
+x+eyrJymBhUL6FVQtoARcYD9g0ya1q3taJQ+JhGW1Ib+DtZzrV4CfDU6q5hWrOOX
+d9/CAPM4NsjxuAfsy8nH+IOmcLyOXgfTgNFYVv5REnLVYOEoE630uBxnrOKchtpk
+1iBSSGCPVcNioLQdUS3rPxtgkZkthar22xme7RDuUj1cg9p6Gu+6hyJIB7y41NdM
+rLdZeHcRlgy56yb6YBXTnilPDCFhtOx6L8cXnL4CVYtg7ityq5khDSMVrtgiF8wQ
+n6hDJbSLdFMQMdm9gIQ6lobZkHi4R3yk9S/rHtl7Gc3Set/2rqnxpyt5WsNHcBoy
+uNkvGZuP9Pb6n4k7eR0/qX2cg3xycNI/uuxqDTpieHr+/lvOflqcj6+6Fq3Uvg65
+8rl5vzsrWArX/3/5sfGG6pqPaCjEHb0FeP8zzxzUTw6J46mzCuG90ERCJ/75wTmT
+QD3oCtLtu/nI4MsR8I4VVn26u8FO63xDSk8xPvS6o8wU7EoZXH3+74EFf5beGgt8
+cMTS1Zil/MrtFOSC+MypihKCaYYjVr66F3h3I1RBef+bwuwOuQacaQCXkLHOWC3S
+pH1iuKGt7lbpGPz103pkc4ssMYAc66nEYXf9I8MATP1aYOyP5o78yegWqgiUs+jd
+frdgEsW3fsmeA655+5XZmXLHlmkpbb31KeVfCQXoTbHvExTqK91k73xn7/YRHLKq
+vFKsz6cuWFnHmhb9gInH8iNzEM8DEJq+lEEhEi9XjeNmgnzd2vVl+3a2GPoy2h7u
+VoGAwr7phI1PiD2aRoB7ZWiR4xxbwl8n+hHh63hSGNYHOeQ7JosPnqcwvHUZo4JZ
+CXAI6T9snlZRg2G/BT627LYRGqu8piWl3FJXVaVd8lo6g4ZUrhyuV+48tJy1OvHT
+gM1IATYnml6FPLXAqouxDrMKToAw45KOLrevGDDaQ91kxPrgEpK3fcnvH0FgJ16x
+/N7uqBmo2XYZM6QxTrq1iShpGFoZ+DC3FOtDT3TKnsrlEUBLzgP3yqJje9Dn+BRs
+td8=
+-----END ENCRYPTED PRIVATE KEY-----

tests/integration/quteprocess.py

@@ -209,7 +209,7 @@ class QuteProc(testprocess.Process):
                  'about:blank']
         return executable, args
 
-    def path_to_url(self, path):
+    def path_to_url(self, path, *, port=None, https=False):
         """Get a URL based on a filename for the localhost webserver.
 
         URLs like about:... and qute:... are handled specially and returned
@@ -218,8 +218,9 @@ class QuteProc(testprocess.Process):
         if path.startswith('about:') or path.startswith('qute:'):
             return path
         else:
-            return 'http://localhost:{}/{}'.format(
+            return '{}://localhost:{}/{}'.format(
-                self._httpbin.port,
+                'https' if https else 'http',
+                self._httpbin.port if port is None else port,
                 path if path != '/' else '')
 
     def after_test(self):
@@ -266,12 +267,13 @@ class QuteProc(testprocess.Process):
         yield
         self.set_setting(sect, opt, old_value)
 
-    def open_path(self, path, new_tab=False, new_window=False):
+    def open_path(self, path, *, new_tab=False, new_window=False, port=None,
+                  https=False):
         """Open the given path on the local webserver in qutebrowser."""
         if new_tab and new_window:
             raise ValueError("new_tab and new_window given!")
 
-        url = self.path_to_url(path)
+        url = self.path_to_url(path, port=port, https=https)
         if new_tab:
             self.send_cmd(':open -t ' + url)
         elif new_window:
@@ -285,9 +287,10 @@ class QuteProc(testprocess.Process):
                              message=message)
         line.expected = True
 
-    def wait_for_load_finished(self, path, timeout=15000):
+    def wait_for_load_finished(self, path, *, port=None, https=False,
+                               timeout=15000):
         """Wait until any tab has finished loading."""
-        url = self.path_to_url(path)
+        url = self.path_to_url(path, port=port, https=https)
         # We really need the same representation that the webview uses in its
         # __repr__
         url = utils.elide(QUrl(url).toDisplayString(QUrl.EncodeUnicode), 100)

tests/integration/test_mhtml_e2e.py

@@ -87,7 +87,10 @@ def test_mhtml(test_name, download_dir, quteproc, httpbin):
                             'data', 'downloads', 'mhtml', test_name)
     test_path = 'data/downloads/mhtml/{}'.format(test_name)
 
-    quteproc.open_path('{}/{}.html'.format(test_path, test_name))
+    url_path = '{}/{}.html'.format(test_path, test_name)
+    quteproc.open_path(url_path)
+    quteproc.wait_for_load_finished(url_path)
+
     download_dest = os.path.join(download_dir.location,
                                  '{}-downloaded.mht'.format(test_name))
 

tests/integration/webserver.py

@@ -19,6 +19,7 @@
 
 """Fixtures for the httpbin webserver."""
 
+import re
 import sys
 import json
 import socket
@@ -94,7 +95,7 @@ class ExpectedRequest:
                 .format(self.verb, self.path))
 
 
-class HTTPBin(testprocess.Process):
+class WebserverProcess(testprocess.Process):
 
     """Abstraction over a running HTTPbin server process.
 
@@ -110,8 +111,9 @@ class HTTPBin(testprocess.Process):
 
     KEYS = ['verb', 'path']
 
-    def __init__(self, parent=None):
+    def __init__(self, script, parent=None):
         super().__init__(parent)
+        self._script = script
         self.port = self._get_port()
         self.new_data.connect(self.new_request)
 
@@ -130,8 +132,9 @@ class HTTPBin(testprocess.Process):
 
     def _parse_line(self, line):
         self._log(line)
-        if line == (' * Running on http://127.0.0.1:{}/ (Press CTRL+C to '
+        started_re = re.compile(r' \* Running on https?://127\.0\.0\.1:{}/ '
-                    'quit)'.format(self.port)):
+                                r'\(Press CTRL\+C to quit\)'.format(self.port))
+        if started_re.fullmatch(line):
             self.ready.emit()
             return None
         return Request(line)
@@ -139,12 +142,12 @@ class HTTPBin(testprocess.Process):
     def _executable_args(self):
         if hasattr(sys, 'frozen'):
             executable = os.path.join(os.path.dirname(sys.executable),
-                                      'webserver_sub')
+                                      self._script)
             args = [str(self.port)]
         else:
             executable = sys.executable
             py_file = os.path.join(os.path.dirname(__file__),
-                                   'webserver_sub.py')
+                                   self._script + '.py')
             args = [py_file, str(self.port)]
         return executable, args
 
@@ -157,7 +160,7 @@ class HTTPBin(testprocess.Process):
 @pytest.yield_fixture(scope='session', autouse=True)
 def httpbin(qapp):
     """Fixture for a httpbin object which ensures clean setup/teardown."""
-    httpbin = HTTPBin()
+    httpbin = WebserverProcess('webserver_sub')
     httpbin.start()
     yield httpbin
     httpbin.cleanup()
@@ -169,3 +172,18 @@ def httpbin_after_test(httpbin, request):
     request.node._httpbin_log = httpbin.captured_log
     yield
     httpbin.after_test()
+
+
+@pytest.yield_fixture
+def ssl_server(request, qapp):
+    """Fixture for a webserver with a self-signed SSL certificate.
+
+    This needs to be explicitly used in a test, and overwrites the httpbin log
+    used in that test.
+    """
+    server = WebserverProcess('webserver_sub_ssl')
+    request.node._httpbin_log = server.captured_log
+    server.start()
+    yield server
+    server.after_test()
+    server.cleanup()

tests/integration/webserver_sub_ssl.py

@@ -0,0 +1,47 @@
+import ssl
+import sys
+import json
+import logging
+import os.path
+
+import flask
+
+
+app = flask.Flask(__name__)
+
+
+@app.route('/')
+def hello_world():
+    return "Hello World via SSL!"
+
+
+@app.after_request
+def log_request(response):
+    """Log a webserver request."""
+    request = flask.request
+    data = {
+        'verb': request.method,
+        'path': request.full_path if request.query_string else request.path,
+        'status': response.status_code,
+    }
+    print(json.dumps(data), file=sys.stderr, flush=True)
+    return response
+
+
+@app.before_first_request
+def turn_off_logging():
+    # Turn off werkzeug logging after the startup message has been printed.
+    logging.getLogger('werkzeug').setLevel(logging.ERROR)
+
+
+def main():
+    ssl_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                           'data', 'ssl')
+    context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+    context.load_cert_chain(os.path.join(ssl_dir, 'cert.pem'),
+                            os.path.join(ssl_dir, 'key.pem'))
+    app.run(port=int(sys.argv[1]), debug=False, ssl_context=context)
+
+
+if __name__ == '__main__':
+    main()