Add a harness that runs tests from tlsfuzzer

This currently runs 54 tests from the tlsfuzzer suite against
the TLSv1.3 server which exercise a large portion of the code.
They already found a number of bugs and misbehaviors and also
inspired a few diffs currently in the pipeline.

This regress requires the py3-tlsfuzzer package to be installed,
otherwise the tests are skipped. Many thanks to kmos for helping
with the ports side and to beck for his positive feedback.

ok beck

2 files changed, 781 insertions, 0 deletions

diff --git a/src/regress/lib/libssl/tlsfuzzer/Makefile b/src/regress/lib/libssl/tlsfuzzer/Makefile
new file mode 100644
index 0000000000..224fc96c5c
--- /dev/null
+++ b/src/regress/lib/libssl/tlsfuzzer/Makefile
@@ -0,0 +1,45 @@
+#       $OpenBSD: Makefile,v 1.1 2020/05/21 10:38:43 tb Exp $
+
+.if !exists(/usr/local/share/tlsfuzzer)
+regress:
+        @echo package py3-tlsfuzzer is required for this regress
+        @echo SKIPPED
+.else
+
+localhost.key localhost.crt:
+        openssl req -x509 -newkey rsa -keyout localhost.key -out localhost.crt \
+                -subj /CN=localhost -nodes -batch
+
+certs: localhost.key localhost.crt
+
+CLEANFILES      += localhost.key localhost.crt
+
+PORT            ?= 4433
+SLOW            = -s
+TIMING          = # -t
+VERBOSE         = # -v
+
+all: certs
+        python3 ${.CURDIR}/tlsfuzzer.py ${SLOW} ${TIMING} ${VERBOSE}
+
+failing: certs
+        python3 ${.CURDIR}/tlsfuzzer.py -f ${SLOW} ${TIMING} ${VERBOSE}
+
+
+port: certs
+        python3 ${.CURDIR}/tlsfuzzer.py ${SLOW} ${TIMING} ${VERBOSE} -p ${PORT}
+
+list:
+        @python3 ${.CURDIR}/tlsfuzzer.py -l
+
+list-failing:
+        @python3 ${.CURDIR}/tlsfuzzer.py -l -f
+
+missing:
+        @python3 ${.CURDIR}/tlsfuzzer.py -m
+
+.PHONY: all certs failing list list-failing missing port
+
+.endif
+
+.include <bsd.regress.mk>
diff --git a/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py b/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py
new file mode 100644
index 0000000000..1098e049f6
--- /dev/null
+++ b/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py
@@ -0,0 +1,736 @@
+#   $OpenBSD: tlsfuzzer.py,v 1.1 2020/05/21 10:38:43 tb Exp $
+#
+# Copyright (c) 2020 Theo Buehler <tb@openbsd.org>
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import getopt
+import os
+import subprocess
+import sys
+from timeit import default_timer as timer
+
+tlsfuzzer_scriptdir = "/usr/local/share/tlsfuzzer/scripts/"
+
+class Test:
+    """
+    Represents a tlsfuzzer test script.
+        name:           the script's name
+        args:           arguments to feed to the script
+        tls12_args:     override args for a TLSv1.2 server
+        tls13_args:     override args for a TLSv1.3 server
+
+    XXX Add client cert support.
+    """
+    def __init__(self, name="", args=[], tls12_args=[], tls13_args=[]):
+        self.name = name
+        self.tls12_args = args
+        self.tls13_args = args
+        if tls12_args:
+            self.tls12_args = tls12_args
+        if tls13_args:
+            self.tls13_args = tls13_args
+
+    def args(self, has_tls1_3: True):
+        if has_tls1_3:
+            return self.tls13_args
+        else:
+            return self.tls12_args
+
+    def __repr__(self):
+        return "<Test: %s tls12_args: %s tls13_args: %s>" % (
+                self.name, self.tls12_args, tls13_args
+            )
+
+class TestGroup:
+    """ A group of Test objects to be run by TestRunner."""
+    def __init__(self, title="Tests", tests=[]):
+        self.title = title
+        self.tests = tests
+
+    def __iter__(self):
+        return iter(self.tests)
+
+# argument to pass to several tests
+tls13_unsupported_ciphers = [
+    "-e", "TLS 1.3 with ffdhe2048",
+    "-e", "TLS 1.3 with ffdhe3072",
+    "-e", "TLS 1.3 with secp521r1",   # XXX: why is this curve problematic?
+    "-e", "TLS 1.3 with x448",
+]
+
+tls13_tests = TestGroup("TLSv1.3 tests", [
+    Test("test-tls13-ccs.py"),
+    Test("test-tls13-conversation.py"),
+    Test("test-tls13-count-tickets.py"),
+    Test("test-tls13-empty-alert.py"),
+    Test("test-tls13-finished-plaintext.py"),
+    Test("test-tls13-hrr.py"),
+    Test("test-tls13-legacy-version.py"),
+    Test("test-tls13-nociphers.py"),
+    Test("test-tls13-record-padding.py"),
+])
+
+# Tests that take a lot of time (> ~30s on an x280)
+tls13_slow_tests = TestGroup("slow TLSv1.3 tests", [
+    # XXX: Investigate the occasional message
+    # "Got shared secret with 1 most significant bytes equal to zero."
+    Test("test-tls13-dhe-shared-secret-padding.py", tls13_unsupported_ciphers),
+
+    Test("test-tls13-invalid-ciphers.py"),
+    Test("test-tls13-serverhello-random.py", tls13_unsupported_ciphers),
+])
+
+tls13_extra_cert_tests = TestGroup("TLSv1.3 certificate tests", [
+    # need to set up client certs to run these
+    Test("test-tls13-certificate-request.py"),
+    Test("test-tls13-certificate-verify.py"),
+    Test("test-tls13-ecdsa-in-certificate-verify.py"),
+
+    # Test expects the server to have installed three certificates:
+    # with P-256, P-384 and P-521 curve. Also SHA1+ECDSA is verified
+    # to not work.
+    Test("test-tls13-ecdsa-support.py"),
+])
+
+tls13_failing_tests = TestGroup("failing TLSv1.3 tests", [
+    # Some tests fail because we fail later than the scripts expect us to.
+    # With X25519, we accept weak peer public keys and fail when we actually
+    # compute the keyshare.  Other tests seem to indicate that we could be
+    # stricter about what keyshares we accept.
+    Test("test-tls13-crfg-curves.py"),
+    Test("test-tls13-ecdhe-curves.py"),
+
+    # Expects a missing_extensions alert
+    # AssertionError: Unexpected message from peer: Handshake(server_hello)
+    Test("test-tls13-keyshare-omitted.py"),
+
+    # https://github.com/openssl/openssl/issues/8369
+    Test("test-tls13-obsolete-curves.py"),
+
+    # 3 failing rsa_pss_pss tests
+    Test("test-tls13-rsa-signatures.py"),
+
+    # AssertionError: Unexpected message from peer: ChangeCipherSpec()
+    # Most failing tests expect the CCS right before finished.
+    # What's up with that?
+    Test("test-tls13-version-negotiation.py"),
+
+    # The following three tests fail due to broken pipe.
+    # AssertionError: Unexpected closure from peer:
+    # 'zero-length app data'
+    # 'zero-length app data with large padding'
+    # 'zero-length app data with padding'
+    Test("test-tls13-zero-length-data.py"),
+])
+
+tls13_slow_failing_tests = TestGroup("slow, failing TLSv1.3 tests", [
+    # After adding record overflow alert, 14 tests fail because
+    # they expect ExpectNewSessionTicket().
+    Test("test-tls13-record-layer-limits.py" ),
+
+    # Other test failures bugs in keyshare/tlsext negotiation?
+    Test("test-tls13-shuffled-extentions.py"),    # should reject 2nd CH
+    Test("test-tls13-unrecognised-groups.py"),    # unexpected closure
+
+    # systematic failures?
+    # TLSBadRecordMAC("Invalid tag, decryption failure")
+    Test("test-tls13-keyupdate.py"),
+    Test("test-tls13-symetric-ciphers.py"),       # unexpected message from peer
+
+    # 70 fail and 644 pass. For some reason the tests expect a decode_error
+    # but we send a decrypt_error after the CBS_mem_equal() fails in
+    # tls13_server_finished_recv() (which is correct).
+    Test("test-tls13-finished.py"),               # decrypt_error -> decode_error?
+
+    # The following two tests fail Test (skip empty extensions for the first one):
+    # 'empty unassigned extensions, ids in range from 2 to 4118'
+    # 'unassigned extensions with random payload, ids in range from 2 to 1046'
+    Test("test-tls13-large-number-of-extensions.py"), # 2 fail: empty/random payload
+
+    # 6 tests fail: 'rsa_pkcs1_{md5,sha{1,224,256,384,512}} signature'
+    # We send server hello, but the test expects handshake_failure
+    Test("test-tls13-pkcs-signature.py"),
+    # 8 tests fail: 'tls13 signature rsa_pss_{pss,rsae}_sha{256,384,512}
+    Test("test-tls13-rsapss-signatures.py"),
+
+    # ExpectNewSessionTicket
+    Test("test-tls13-session-resumption.py"),
+])
+
+tls13_unsupported_tests = TestGroup("TLSv1.3 tests for unsupported features", [
+    # Tests for features we don't support
+    Test("test-tls13-0rtt-garbage.py"),
+    Test("test-tls13-ffdhe-groups.py"),
+    Test("test-tls13-ffdhe-sanity.py"),
+    Test("test-tls13-psk_dhe_ke.py"),
+    Test("test-tls13-psk_ke.py"),
+
+    # need server to react to HTTP GET for /keyupdate
+    Test("test-tls13-keyupdate-from-server.py"),
+
+    # Weird test: tests servers that don't support 1.3
+    Test("test-tls13-non-support.py"),
+
+    # broken test script
+    # UnboundLocalError: local variable 'cert' referenced before assignment
+    Test("test-tls13-post-handshake-auth.py"),
+
+    # Server must be configured to support only rsa_pss_rsae_sha512
+    Test("test-tls13-signature-algorithms.py"),
+])
+
+tls12_exclude_legacy_protocols = [
+    # all these have BIO_read timeouts against TLSv1.3
+    "-e", "Protocol (3, 0)",
+    "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello",
+    # the following only fail with TLSv1.3
+    "-e", "Protocol (3, 1) in SSLv2 compatible ClientHello",
+    "-e", "Protocol (3, 2) in SSLv2 compatible ClientHello",
+    "-e", "Protocol (3, 3) in SSLv2 compatible ClientHello",
+    "-e", "Protocol (3, 1) with secp521r1 group",   # XXX
+    "-e", "Protocol (3, 1) with x448 group",
+    "-e", "Protocol (3, 2) with secp521r1 group",   # XXX
+    "-e", "Protocol (3, 2) with x448 group",
+    "-e", "Protocol (3, 3) with secp521r1 group",   # XXX
+    "-e", "Protocol (3, 3) with x448 group",
+]
+
+tls12_tests = TestGroup("TLSv1.2 tests", [
+    # Tests that pass as they are.
+    Test("test-TLSv1_2-rejected-without-TLSv1_2.py"),
+    Test("test-aes-gcm-nonces.py"),
+    Test("test-chacha20.py"),
+    Test("test-conversation.py"),
+    Test("test-cve-2016-2107.py"),
+    Test("test-dhe-rsa-key-exchange.py"),
+    Test("test-early-application-data.py"),
+    Test("test-empty-extensions.py"),
+    Test("test-fuzzed-MAC.py"),
+    Test("test-fuzzed-ciphertext.py"),
+    Test("test-fuzzed-finished.py"),
+    Test("test-fuzzed-padding.py"),
+    Test("test-hello-request-by-client.py"),
+    Test("test-invalid-cipher-suites.py"),
+    Test("test-invalid-content-type.py"),
+    Test("test-invalid-session-id.py"),
+    Test("test-invalid-version.py"),
+    Test("test-message-skipping.py"),
+    Test("test-no-heartbeat.py"),
+    Test("test-sessionID-resumption.py"),
+    Test("test-sslv2-connection.py"),
+    Test("test-truncating-of-finished.py"),
+    Test("test-truncating-of-kRSA-client-key-exchange.py"),
+    Test("test-unsupported-curve-fallback.py"),
+    Test("test-version-numbers.py"),
+    Test("test-zero-length-data.py"),
+
+    # Tests that need tweaking for unsupported features and ciphers.
+    Test(
+        "test-atypical-padding.py", [
+            "-e", "sanity - encrypt then MAC",
+            "-e", "2^14 bytes of AppData with 256 bytes of padding (SHA1 + Encrypt then MAC)",
+        ]
+    ),
+    Test(
+        "test-dhe-rsa-key-exchange-signatures.py", [
+            "-e", "TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA sha224 signature",
+            "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 sha224 signature",
+            "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA sha224 signature",
+            "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 sha224 signature",
+            "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA sha224 signature",
+        ]
+    ),
+    Test("test-dhe-key-share-random.py", tls12_exclude_legacy_protocols),
+    Test("test-export-ciphers-rejected.py", ["--min-ver", "TLSv1.0"]),
+    Test(
+        "test-downgrade-protection.py",
+        tls12_args = ["--server-max-protocol", "TLSv1.2"],
+        tls13_args = ["--server-max-protocol", "TLSv1.3"],
+    ),
+    Test("test-fallback-scsv.py", tls13_args = ["--tls-1.3"] ),
+    Test("test-serverhello-random.py", args = tls12_exclude_legacy_protocols),
+])
+
+tls12_slow_tests = TestGroup("slow TLSv1.2 tests", [
+    Test("test-cve-2016-7054.py"),
+    Test("test-dhe-no-shared-secret-padding.py", tls12_exclude_legacy_protocols),
+    Test("test-ecdhe-padded-shared-secret.py", tls12_exclude_legacy_protocols),
+    Test("test-ecdhe-rsa-key-share-random.py", tls12_exclude_legacy_protocols),
+    # This test has some failures once in a while.
+    Test("test-fuzzed-plaintext.py"),
+])
+
+tls12_failing_tests = TestGroup("failing TLSv1.2 tests", [
+    # no shared cipher
+    Test("test-aesccm.py"),
+    # need server to set up alpn
+    Test("test-alpn-negotiation.py"),
+    # many tests fail due to unexpected server_name extension
+    Test("test-bleichenbacher-workaround.py"),
+
+    # need client key and cert plus extra server setup
+    Test("test-certificate-malformed.py"),
+    Test("test-certificate-request.py"),
+    Test("test-certificate-verify-malformed-sig.py"),
+    Test("test-certificate-verify-malformed.py"),
+    Test("test-certificate-verify.py"),
+    Test("test-ecdsa-in-certificate-verify.py"),
+    Test("test-renegotiation-disabled-client-cert.py"),
+    Test("test-rsa-pss-sigs-on-certificate-verify.py"),
+    Test("test-rsa-sigs-on-certificate-verify.py"),
+
+    # test doesn't expect session ticket
+    Test("test-client-compatibility.py"),
+    # abrupt closure
+    Test("test-client-hello-max-size.py"),
+    # unknown signature algorithms
+    Test("test-clienthello-md5.py"),
+    # abrupt closure
+    Test("test-cve-2016-6309.py"),
+
+    # failing tests are fixed by sending illegal_parameter alert after
+    # DH_compute_keyTest() in ssl_srvr.c
+    Test("test-dhe-rsa-key-exchange-with-bad-messages.py"),
+    # Tests expect an illegal_parameter alert
+    Test("test-ecdhe-rsa-key-exchange-with-bad-messages.py"),
+
+    # We send a handshake_failure while the test expects to succeed
+    Test("test-ecdhe-rsa-key-exchange.py"),
+
+    # unsupported?
+    Test("test-extended-master-secret-extension-with-client-cert.py"),
+
+    # no shared cipher
+    Test("test-ecdsa-sig-flexibility.py"),
+
+    # unsupported
+    Test("test-encrypt-then-mac-renegotiation.py"),
+    Test("test-encrypt-then-mac.py"),
+    Test("test-extended-master-secret-extension.py"),
+    Test("test-ffdhe-negotiation.py"),
+    # unsupported. Expects the server to send the heartbeat extension
+    Test("test-heartbeat.py"),
+
+    # 29 succeed, 263 fail:
+    #   'n extensions', 'n extensions last empty' n in 4086, 4096, 8192, 16383
+    #   'fuzz ext length to n' n in [0..255] with the exception of 41...
+    Test("test-extensions.py"),
+
+    # Tests expect SH but we send unexpected_message or handshake_failure
+    #   'Application data inside Client Hello'
+    #   'Application data inside Client Key Exchange'
+    #   'Application data inside Finished'
+    Test("test-interleaved-application-data-and-fragmented-handshakes-in-renegotiation.py"),
+    # Tests expect SH but we send handshake_failure
+    #   'Application data before Change Cipher Spec'
+    #   'Application data before Client Key Exchange'
+    #   'Application data before Finished'
+    Test("test-interleaved-application-data-in-renegotiation.py"),
+
+    # broken test script
+    # TypeError: '<' not supported between instances of 'int' and 'NoneType'
+    Test("test-invalid-client-hello-w-record-overflow.py"),
+
+    # Lots of failures. abrupt closure
+    Test("test-invalid-client-hello.py"),
+
+    # Test expects illegal_parameter, we send decode_error in ssl_srvr.c:1016
+    # Need to check that this is correct.
+    Test("test-invalid-compression-methods.py"),
+
+    # abrupt closure
+    # 'encrypted premaster set to all zero (n)' n in 256 384 512
+    Test("test-invalid-rsa-key-exchange-messages.py"),
+
+    # test expects illegal_parameter, we send unrecognized_name (which seems
+    # correct according to rfc 6066?)
+    Test("test-invalid-server-name-extension-resumption.py"),
+    # let through some server names without sending an alert
+    # again illegal_parameter vs unrecognized_name
+    Test("test-invalid-server-name-extension.py"),
+
+    Test("test-large-hello.py"),
+
+    # 14 pass
+    # 7 fail
+    # 'n extensions', n in 4095, 4096, 4097, 8191, 8192, 8193, 16383,
+    Test("test-large-number-of-extensions.py"),
+
+    # 4 failures:
+    #   'insecure (legacy) renegotiation with GET after 2nd handshake'
+    #   'insecure (legacy) renegotiation with incomplete GET'
+    #   'secure renegotiation with GET after 2nd handshake'
+    #   'secure renegotiation with incomplete GET'
+    Test("test-legacy-renegotiation.py"),
+
+    # 1 failure (timeout): we don't send the unexpected_message alert
+    # 'duplicate change cipher spec after Finished'
+    Test("test-message-duplication.py"),
+
+    # server should send status_request
+    Test("test-ocsp-stapling.py"),
+
+    # unexpected closure
+    Test("test-openssl-3712.py"),
+
+    # 3 failures:
+    # 'big, needs fragmentation: max fragment - 16336B extension'
+    # 'big, needs fragmentation: max fragment - 32768B extension'
+    # 'maximum size: max fragment - 65531B extension'
+    Test("test-record-layer-fragmentation.py"),
+
+    # wants --reply-AD-size
+    Test("test-record-size-limit.py"),
+
+    # failed: 3 (expect an alert, we send AD)
+    # 'try insecure (legacy) renegotiation with incomplete GET'
+    # 'try secure renegotiation with GET after 2nd CH'
+    # 'try secure renegotiation with incomplete GET'
+    Test("test-renegotiation-disabled.py"),
+
+    # 'resumption of safe session with NULL cipher'
+    # 'resumption with cipher from old CH but not selected by server'
+    Test("test-resumption-with-wrong-ciphers.py"),
+
+    # 5 failures:
+    #   'empty sigalgs'
+    #   'only undefined sigalgs'
+    #   'rsa_pss_pss_sha256 only'
+    #   'rsa_pss_pss_sha384 only'
+    #   'rsa_pss_pss_sha512 only'
+    Test("test-sig-algs.py"),
+
+    # 13 failures:
+    #   'duplicated n non-rsa schemes' for n in 202 2342 8119 23741 32744
+    #   'empty list of signature methods'
+    #   'tolerance n RSA or ECDSA methods' for n in 215 2355 8132 23754
+    #   'tolerance 32758 methods with sig_alg_cert'
+    #   'tolerance max 32744 number of methods with sig_alg_cert'
+    #   'tolerance max (32760) number of methods'
+    Test("test-signature-algorithms.py"),
+
+    # times out
+    Test("test-ssl-death-alert.py"),
+
+    # 17 pass, 13 fail. padding and truncation
+    Test("test-truncating-of-client-hello.py"),
+
+    # x448 tests need disabling plus x25519 corner cases need sorting out
+    Test("test-x25519.py"),
+])
+
+tls12_unsupported_tests = TestGroup("TLSv1.2 for unsupported features", [
+    # protocol_version
+    Test("test-SSLv3-padding.py"),
+])
+
+# These tests take a ton of time to fail against an 1.3 server,
+# so don't run them against 1.3 pending further investigation.
+legacy_tests = TestGroup("Legacy protocol tests", [
+    Test("test-sslv2-force-cipher-3des.py"),
+    Test("test-sslv2-force-cipher-non3des.py"),
+    Test("test-sslv2-force-cipher.py"),
+    Test("test-sslv2-force-export-cipher.py"),
+    Test("test-sslv2hello-protocol.py"),
+])
+
+all_groups = [
+    tls13_tests,
+    tls13_slow_tests,
+    tls13_extra_cert_tests,
+    tls13_failing_tests,
+    tls13_slow_failing_tests,
+    tls13_unsupported_tests,
+    tls12_tests,
+    tls12_slow_tests,
+    tls12_failing_tests,
+    tls12_unsupported_tests,
+    legacy_tests,
+]
+
+failing_groups = [
+    tls13_failing_tests,
+    tls13_slow_failing_tests,
+    tls12_failing_tests,
+]
+
+class TestRunner:
+    """ Runs the given tests troups against a server and displays stats. """
+
+    def __init__(
+        self, timing=False, verbose=False, port=4433, use_tls1_3=True,
+        dry_run=False, tests=[], scriptdir=tlsfuzzer_scriptdir,
+    ):
+        self.tests = []
+
+        self.dryrun = dry_run
+        self.use_tls1_3 = use_tls1_3
+        self.port = str(port)
+        self.scriptdir = scriptdir
+
+        self.stats = []
+        self.failed = []
+
+        self.timing = timing
+        self.verbose = verbose
+
+    def add(self, title="tests", tests=[]):
+        # tests.sort(key=lambda test: test.name)
+        self.tests.append(TestGroup(title, tests))
+
+    def add_group(self, group):
+        self.tests.append(group)
+
+    def run_script(self, test):
+        script = test.name
+        args = ["-p"] + [self.port] + test.args(self.use_tls1_3)
+
+        if self.dryrun:
+            if not self.verbose:
+                args = []
+            print(script , end=' ' if args else '')
+            print(' '.join([f"\"{arg}\"" for arg in args]))
+            return
+
+        if self.verbose:
+            print(script)
+        else:
+            print(f"{script[:68]:<72}", end=" ", flush=True)
+        start = timer()
+        test = subprocess.run(
+            ["python3", os.path.join(self.scriptdir, script)] + args,
+            capture_output=not self.verbose,
+            text=True,
+        )
+        end = timer()
+        self.stats.append((script, end - start))
+        if test.returncode == 0:
+            print("OK")
+            return
+        print("FAILED")
+        self.failed.append(script)
+
+        if self.verbose:
+            return
+
+        print('\n'.join(test.stdout.split("Test end\n", 1)[1:]), end="")
+
+    def run(self):
+        for group in self:
+            print(f"Running {group.title} ...")
+            for test in group:
+                self.run_script(test)
+        return not self.failed
+
+    def __iter__(self):
+        return iter(self.tests)
+
+    def __del__(self):
+        if self.timing and self.stats:
+            total = 0.0
+            for (script, time) in self.stats:
+                print(f"{round(time, 2):6.2f} {script}")
+                total += time
+            print(f"{round(total, 2):6.2f} total")
+
+        if self.failed:
+            print("Failed tests:")
+            print('\n'.join(self.failed))
+
+class TlsServer:
+    """ Spawns an s_server listening on localhost:port if necessary. """
+
+    def __init__(self, port=4433):
+        self.spawn = True
+        # Check whether a server is already listening on localhost:port
+        self.spawn = subprocess.run(
+            ["nc", "-c", "-z", "-T", "noverify", "localhost", str(port)],
+            stderr=subprocess.DEVNULL,
+        ).returncode != 0
+
+        if self.spawn:
+            self.server = subprocess.Popen(
+                [
+                    "openssl",
+                    "s_server",
+                    "-accept",
+                    str(port),
+                    "-key",
+                    "localhost.key",
+                    "-cert",
+                    "localhost.crt",
+                    "-www",
+                ],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+
+        # Check whether the server talks TLSv1.3
+        self.has_tls1_3 = subprocess.run(
+            [
+                "nc",
+                "-c",
+                "-z",
+                "-T",
+                "noverify",
+                "-T",
+                "protocols=TLSv1.3",
+                "localhost",
+                str(port),
+            ],
+            stderr=subprocess.DEVNULL,
+        ).returncode == 0
+
+        self.check()
+
+    def check(self):
+        if self.spawn and self.server.poll() is not None:
+            print(self.server.stderr.read())
+            raise RuntimeError(
+                f"openssl s_server died. Return code: {self.server.returncode}."
+            )
+        if self.spawn:
+            self.server.stderr.detach()
+
+    def __del__(self):
+        if self.spawn:
+            self.server.terminate()
+
+# Extract the arguments we pass to script
+def defaultargs(script, has_tls1_3):
+    return next(
+        (test for group in all_groups for test in group if test.name == script),
+        Test()
+    ).args(has_tls1_3)
+
+def list_or_missing(missing=True):
+    tests = [test.name for group in all_groups for test in group]
+
+    if missing:
+        scripts = {
+            f for f in os.listdir(tlsfuzzer_scriptdir) if f != "__pycache__"
+        }
+        missing = scripts - set(tests)
+        if missing:
+            print('\n'.join(sorted(missing)))
+        exit(0)
+
+    tests.sort()
+    print('\n'.join(tests))
+    exit(0)
+
+def usage():
+    print("Usage: python3 tlsfuzzer.py [-lmnstv] [-p port] [script [test...]]")
+    print(" --help      help")
+    print(" -f          run failing tests")
+    print(" -l          list tests")
+    print(" -m          list new tests after package update")
+    print(" -n          do not run tests, but list the ones that would be run")
+    print(" -p port     connect to this port - defaults to 4433")
+    print(" -s          run slow tests")
+    print(" -t          show timing stats at end")
+    print(" -v          verbose output")
+    exit(0)
+
+def main():
+    failing = False
+    list = False
+    missing = False
+    dryrun = False
+    port = 4433
+    slow = False
+    timing = False
+    verbose = False
+
+    argv = sys.argv[1:]
+    opts, args = getopt.getopt(argv, "flmnp:stv", ["help"])
+    for opt, arg in opts:
+        if opt == '--help':
+            usage()
+        elif opt == '-f':
+            failing = True
+        elif opt == '-l':
+            list = True
+        elif opt == '-m':
+            missing = True
+        elif opt == '-n':
+            dryrun = True
+        elif opt == '-p':
+            port = int(arg)
+        elif opt == '-s':
+            slow = True
+        elif opt == '-t':
+            timing = True
+        elif opt == '-v':
+            verbose = True
+        else:
+            raise ValueError(f"Unknown option: {opt}")
+
+    if not os.path.exists(tlsfuzzer_scriptdir):
+        print("package py3-tlsfuzzer is required for this regress")
+        exit(1)
+
+    if list and failing:
+        failing = [test.name for group in failing_groups for test in group]
+        failing.sort()
+        print('\n'.join(failing))
+        exit(0)
+
+    if list or missing:
+        list_or_missing(missing)
+
+    tls_server = TlsServer(port)
+
+    tests = TestRunner(timing, verbose, port, tls_server.has_tls1_3, dryrun)
+
+    if args:
+        (dir, script) = os.path.split(args[0])
+        if dir and not dir == '.':
+            tests.scriptdir = dir
+
+        testargs = defaultargs(script, tls_server.has_tls1_3)
+
+        tests.verbose = True
+        tests.add("test from command line", [Test(script, testargs + args[1:])])
+
+        exit(not tests.run())
+
+    if failing:
+        if tls_server.has_tls1_3:
+            tests.add_group(tls13_failing_tests)
+            if slow:
+                tests.add_group(tls13_slow_failing_tests)
+        tests.add_group(tls12_failing_tests)
+
+    if tls_server.has_tls1_3:
+        tests.add_group(tls13_tests)
+        if slow:
+            tests.add_group(tls13_slow_tests)
+    else:
+        tests.add_group(legacy_tests)
+
+    tests.add_group(tls12_tests)
+    if slow:
+        tests.add_group(tls12_slow_tests)
+
+    success = tests.run()
+    del tests
+
+    if not success:
+        print("FAILED")
+        exit(1)
+
+if __name__ == "__main__":
+    main()