summaryrefslogtreecommitdiff
path: root/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py
diff options
context:
space:
mode:
Diffstat (limited to 'src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py')
-rw-r--r--src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py921
1 files changed, 0 insertions, 921 deletions
diff --git a/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py b/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py
deleted file mode 100644
index aa7e384e1f..0000000000
--- a/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py
+++ /dev/null
@@ -1,921 +0,0 @@
1# $OpenBSD: tlsfuzzer.py,v 1.50 2023/07/02 17:21:33 beck Exp $
2#
3# Copyright (c) 2020 Theo Buehler <tb@openbsd.org>
4#
5# Permission to use, copy, modify, and distribute this software for any
6# purpose with or without fee is hereby granted, provided that the above
7# copyright notice and this permission notice appear in all copies.
8#
9# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16
17import getopt
18import os
19import subprocess
20import sys
21from timeit import default_timer as timer
22
23tlsfuzzer_scriptdir = "/usr/local/share/tlsfuzzer/scripts/"
24
25class Test:
26 """
27 Represents a tlsfuzzer test script.
28 name: the script's name
29 args: arguments to feed to the script
30 tls12_args: override args for a TLSv1.2 server
31 tls13_args: override args for a TLSv1.3 server
32
33 XXX Add client cert support.
34 """
35 def __init__(self, name="", args=[], tls12_args=[], tls13_args=[]):
36 self.name = name
37 self.tls12_args = args
38 self.tls13_args = args
39 if tls12_args:
40 self.tls12_args = tls12_args
41 if tls13_args:
42 self.tls13_args = tls13_args
43
44 def args(self, has_tls1_3: True):
45 if has_tls1_3:
46 return self.tls13_args
47 else:
48 return self.tls12_args
49
50 def __repr__(self):
51 return "<Test: %s tls12_args: %s tls13_args: %s>" % (
52 self.name, self.tls12_args, self.tls13_args
53 )
54
55class TestGroup:
56 """ A group of Test objects to be run by TestRunner."""
57 def __init__(self, title="Tests", tests=[]):
58 self.title = title
59 self.tests = tests
60
61 def __iter__(self):
62 return iter(self.tests)
63
64# argument to pass to several tests
65tls13_unsupported_ciphers = [
66 "-e", "TLS 1.3 with ffdhe2048",
67 "-e", "TLS 1.3 with ffdhe3072",
68 "-e", "TLS 1.3 with x448",
69]
70
71def substitute_alert(want, got):
72 return f"Expected alert description \"{want}\" " \
73 + f"does not match received \"{got}\""
74
75# test-tls13-finished.py has 70 failing tests that expect a "decode_error"
76# instead of the "decrypt_error" sent by tls13_server_finished_recv().
77# Both alerts appear to be reasonable in this context, so work around this
78# in the test instead of the library.
79def generate_test_tls13_finished_args():
80 assertion = substitute_alert("decode_error", "decrypt_error");
81 paddings = [
82 ("TLS_AES_128_GCM_SHA256", 0, 1),
83 ("TLS_AES_128_GCM_SHA256", 0, 2),
84 ("TLS_AES_128_GCM_SHA256", 0, 4),
85 ("TLS_AES_128_GCM_SHA256", 0, 8),
86 ("TLS_AES_128_GCM_SHA256", 0, 16),
87 ("TLS_AES_128_GCM_SHA256", 0, 32),
88 ("TLS_AES_128_GCM_SHA256", 0, 48),
89 ("TLS_AES_128_GCM_SHA256", 0, 2**14-4-32),
90 ("TLS_AES_128_GCM_SHA256", 0, 0x20000),
91 ("TLS_AES_128_GCM_SHA256", 0, 0x30000),
92 ("TLS_AES_128_GCM_SHA256", 1, 0),
93 ("TLS_AES_128_GCM_SHA256", 2, 0),
94 ("TLS_AES_128_GCM_SHA256", 4, 0),
95 ("TLS_AES_128_GCM_SHA256", 8, 0),
96 ("TLS_AES_128_GCM_SHA256", 16, 0),
97 ("TLS_AES_128_GCM_SHA256", 32, 0),
98 ("TLS_AES_128_GCM_SHA256", 48, 0),
99 ("TLS_AES_128_GCM_SHA256", 2**14-4-32, 0),
100 ("TLS_AES_128_GCM_SHA256", 12, 0),
101 ("TLS_AES_128_GCM_SHA256", 1, 1),
102 ("TLS_AES_128_GCM_SHA256", 8, 8),
103 ("TLS_AES_256_GCM_SHA384", 0, 1),
104 ("TLS_AES_256_GCM_SHA384", 0, 2),
105 ("TLS_AES_256_GCM_SHA384", 0, 4),
106 ("TLS_AES_256_GCM_SHA384", 0, 8),
107 ("TLS_AES_256_GCM_SHA384", 0, 16),
108 ("TLS_AES_256_GCM_SHA384", 0, 32),
109 ("TLS_AES_256_GCM_SHA384", 0, 48),
110 ("TLS_AES_256_GCM_SHA384", 0, 2**14-4-48),
111 ("TLS_AES_256_GCM_SHA384", 0, 0x20000),
112 ("TLS_AES_256_GCM_SHA384", 0, 0x30000),
113 ("TLS_AES_256_GCM_SHA384", 0, 12),
114 ("TLS_AES_256_GCM_SHA384", 1, 0),
115 ("TLS_AES_256_GCM_SHA384", 2, 0),
116 ("TLS_AES_256_GCM_SHA384", 4, 0),
117 ("TLS_AES_256_GCM_SHA384", 8, 0),
118 ("TLS_AES_256_GCM_SHA384", 16, 0),
119 ("TLS_AES_256_GCM_SHA384", 32, 0),
120 ("TLS_AES_256_GCM_SHA384", 48, 0),
121 ("TLS_AES_256_GCM_SHA384", 2**14-4-48, 0),
122 ("TLS_AES_256_GCM_SHA384", 1, 1),
123 ("TLS_AES_256_GCM_SHA384", 8, 8),
124 ]
125 truncations = [
126 ("TLS_AES_128_GCM_SHA256", 0, -1),
127 ("TLS_AES_128_GCM_SHA256", 0, -2),
128 ("TLS_AES_128_GCM_SHA256", 0, -4),
129 ("TLS_AES_128_GCM_SHA256", 0, -8),
130 ("TLS_AES_128_GCM_SHA256", 0, -16),
131 ("TLS_AES_128_GCM_SHA256", 0, -32),
132 ("TLS_AES_128_GCM_SHA256", 0, 12),
133 ("TLS_AES_128_GCM_SHA256", 1, None),
134 ("TLS_AES_128_GCM_SHA256", 2, None),
135 ("TLS_AES_128_GCM_SHA256", 4, None),
136 ("TLS_AES_128_GCM_SHA256", 8, None),
137 ("TLS_AES_128_GCM_SHA256", 16, None),
138 ("TLS_AES_128_GCM_SHA256", 32, None),
139 ("TLS_AES_256_GCM_SHA384", 0, -1),
140 ("TLS_AES_256_GCM_SHA384", 0, -2),
141 ("TLS_AES_256_GCM_SHA384", 0, -4),
142 ("TLS_AES_256_GCM_SHA384", 0, -8),
143 ("TLS_AES_256_GCM_SHA384", 0, -16),
144 ("TLS_AES_256_GCM_SHA384", 0, -32),
145 ("TLS_AES_256_GCM_SHA384", 0, 12),
146 ("TLS_AES_256_GCM_SHA384", 1, None),
147 ("TLS_AES_256_GCM_SHA384", 2, None),
148 ("TLS_AES_256_GCM_SHA384", 4, None),
149 ("TLS_AES_256_GCM_SHA384", 8, None),
150 ("TLS_AES_256_GCM_SHA384", 16, None),
151 ("TLS_AES_256_GCM_SHA384", 32, None),
152 ]
153
154 args = [
155 "-x", "empty - cipher TLS_AES_128_GCM_SHA256", "-X", assertion,
156 "-x", "empty - cipher TLS_AES_256_GCM_SHA384", "-X", assertion,
157 ]
158 padding_fmt = "padding - cipher %s, pad_byte 0, pad_left %d, pad_right %d"
159 for padding in paddings:
160 args += ["-x", padding_fmt % padding, "-X", assertion]
161 truncation_fmt = "truncation - cipher %s, start %d, end %s"
162 for truncation in truncations:
163 args += ["-x", truncation_fmt % truncation, "-X", assertion]
164 return args
165
166tls13_tests = TestGroup("TLSv1.3 tests", [
167 Test("test-tls13-ccs.py"),
168 Test("test-tls13-conversation.py"),
169 Test("test-tls13-count-tickets.py"),
170 Test("test-tls13-empty-alert.py"),
171 Test("test-tls13-finished.py", generate_test_tls13_finished_args()),
172 Test("test-tls13-finished-plaintext.py"),
173 Test("test-tls13-hrr.py"),
174 Test("test-tls13-keyshare-omitted.py"),
175 Test("test-tls13-legacy-version.py"),
176 Test("test-tls13-nociphers.py"),
177 Test("test-tls13-record-padding.py"),
178 # Exclude QUIC transport parameters
179 Test("test-tls13-shuffled-extentions.py", [ "--exc", "57" ]),
180 Test("test-tls13-zero-content-type.py"),
181
182 # The skipped tests fail due to a bug in BIO_gets() which masks the retry
183 # signalled from an SSL_read() failure. Testing with httpd(8) shows we're
184 # handling these corner cases correctly since tls13_record_layer.c -r1.47.
185 Test("test-tls13-zero-length-data.py", [
186 "-e", "zero-length app data",
187 "-e", "zero-length app data with large padding",
188 "-e", "zero-length app data with padding",
189 ]),
190
191 # We don't currently handle NSTs
192 Test("test-tls13-connection-abort.py", ["-e", "After NewSessionTicket"]),
193])
194
195# Tests that take a lot of time (> ~30s on an x280)
196tls13_slow_tests = TestGroup("slow TLSv1.3 tests", [
197 # XXX: Investigate the occasional message
198 # "Got shared secret with 1 most significant bytes equal to zero."
199 Test("test-tls13-dhe-shared-secret-padding.py", tls13_unsupported_ciphers),
200
201 Test("test-tls13-invalid-ciphers.py"),
202 Test("test-tls13-serverhello-random.py", tls13_unsupported_ciphers),
203
204 # Mark two tests cases as xfail for now. The tests expect an arguably
205 # correct decode_error while we send a decrypt_error (like fizz/boring).
206 Test("test-tls13-record-layer-limits.py", [
207 "-x", "max size payload (2**14) of Finished msg, with 16348 bytes of left padding, cipher TLS_AES_128_GCM_SHA256",
208 "-X", substitute_alert("decode_error", "decrypt_error"),
209 "-x", "max size payload (2**14) of Finished msg, with 16348 bytes of left padding, cipher TLS_CHACHA20_POLY1305_SHA256",
210 "-X", substitute_alert("decode_error", "decrypt_error"),
211 ]),
212 # We don't accept an empty ECPF extension since it must advertise the
213 # uncompressed point format. Exclude this extension type from the test.
214 # Also exclude QUIC transport parameters.
215 Test(
216 "test-tls13-large-number-of-extensions.py",
217 tls13_args = ["--exc", "11", "--exc", "57"],
218 ),
219])
220
221tls13_extra_cert_tests = TestGroup("TLSv1.3 certificate tests", [
222 # need to set up client certs to run these
223 Test("test-tls13-certificate-request.py"),
224 Test("test-tls13-certificate-verify.py"),
225 Test("test-tls13-ecdsa-in-certificate-verify.py"),
226 Test("test-tls13-eddsa-in-certificate-verify.py"),
227
228 # Test expects the server to have installed three certificates:
229 # with P-256, P-384 and P-521 curve. Also SHA1+ECDSA is verified
230 # to not work.
231 Test("test-tls13-ecdsa-support.py"),
232])
233
234tls13_failing_tests = TestGroup("failing TLSv1.3 tests", [
235 # Some tests fail because we fail later than the scripts expect us to.
236 # With X25519, we accept weak peer public keys and fail when we actually
237 # compute the keyshare. Other tests seem to indicate that we could be
238 # stricter about what keyshares we accept.
239 Test("test-tls13-crfg-curves.py", [
240 '-e', 'all zero x448 key share',
241 '-e', 'empty x448 key share',
242 '-e', 'sanity x448 with compression ansiX962_compressed_char2',
243 '-e', 'sanity x448 with compression ansiX962_compressed_prime',
244 '-e', 'sanity x448 with compression uncompressed',
245 '-e', 'too big x448 key share',
246 '-e', 'too small x448 key share',
247 '-e', 'x448 key share of "1"',
248 ]),
249 Test("test-tls13-ecdhe-curves.py", [
250 '-e', 'sanity - x448',
251 '-e', 'x448 - key share from other curve',
252 '-e', 'x448 - point at infinity',
253 '-e', 'x448 - right 0-padded key_share',
254 '-e', 'x448 - right-truncated key_share',
255 ]),
256
257 # The test sends records with protocol version 0x0300 instead of 0x0303
258 # and currently fails with OpenSSL and LibreSSL for this reason.
259 # We have the logic corresponding to NSS's fix for CVE-2020-25648
260 # https://hg.mozilla.org/projects/nss/rev/57bbefa793232586d27cee83e74411171e128361
261 # so should not be affected by this issue.
262 Test("test-tls13-multiple-ccs-messages.py"),
263
264 # https://github.com/openssl/openssl/issues/8369
265 Test("test-tls13-obsolete-curves.py"),
266
267 # 3 failing rsa_pss_pss tests
268 Test("test-tls13-rsa-signatures.py"),
269
270 # The failing tests all expect an ri extension. What's up with that?
271 Test("test-tls13-version-negotiation.py"),
272])
273
274tls13_slow_failing_tests = TestGroup("slow, failing TLSv1.3 tests", [
275 # Other test failures bugs in keyshare/tlsext negotiation?
276 Test("test-tls13-unrecognised-groups.py"), # unexpected closure
277
278 # 5 occasional failures:
279 # 'app data split, conversation with KeyUpdate msg'
280 # 'fragmented keyupdate msg'
281 # 'multiple KeyUpdate messages'
282 # 'post-handshake KeyUpdate msg with update_not_request'
283 # 'post-handshake KeyUpdate msg with update_request'
284 Test("test-tls13-keyupdate.py"),
285
286 Test("test-tls13-symetric-ciphers.py"), # unexpected message from peer
287
288 # 6 tests fail: 'rsa_pkcs1_{md5,sha{1,224,256,384,512}} signature'
289 # We send server hello, but the test expects handshake_failure
290 Test("test-tls13-pkcs-signature.py"),
291 # 8 tests fail: 'tls13 signature rsa_pss_{pss,rsae}_sha{256,384,512}
292 Test("test-tls13-rsapss-signatures.py"),
293])
294
295tls13_unsupported_tests = TestGroup("TLSv1.3 tests for unsupported features", [
296 # Tests for features we don't support
297 Test("test-tls13-0rtt-garbage.py"),
298 Test("test-tls13-ffdhe-groups.py"),
299 Test("test-tls13-ffdhe-sanity.py"),
300 Test("test-tls13-psk_dhe_ke.py"),
301 Test("test-tls13-psk_ke.py"),
302
303 # need server to react to HTTP GET for /keyupdate
304 Test("test-tls13-keyupdate-from-server.py"),
305
306 # needs an echo server
307 Test("test-tls13-lengths.py"),
308
309 # Weird test: tests servers that don't support 1.3
310 Test("test-tls13-non-support.py"),
311
312 # broken test script
313 # UnboundLocalError: local variable 'cert' referenced before assignment
314 Test("test-tls13-post-handshake-auth.py"),
315
316 # ExpectNewSessionTicket
317 Test("test-tls13-session-resumption.py"),
318
319 # Server must be configured to support only rsa_pss_rsae_sha512
320 Test("test-tls13-signature-algorithms.py"),
321])
322
323tls12_exclude_legacy_protocols = [
324 # all these have BIO_read timeouts against TLSv1.3
325 "-e", "Protocol (3, 0)",
326 "-e", "Protocol (3, 1)",
327 "-e", "Protocol (3, 2)",
328 "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello",
329 # the following only fail with TLSv1.3
330 "-e", "Protocol (3, 1) in SSLv2 compatible ClientHello",
331 "-e", "Protocol (3, 2) in SSLv2 compatible ClientHello",
332 "-e", "Protocol (3, 3) in SSLv2 compatible ClientHello",
333 "-e", "Protocol (3, 1) with x448 group",
334 "-e", "Protocol (3, 2) with x448 group",
335 "-e", "Protocol (3, 3) with x448 group",
336 # These don't work without TLSv1.0 and TLSv1.1
337 "-e", "Protocol (3, 1) with secp256r1 group",
338 "-e", "Protocol (3, 1) with secp384r1 group",
339 "-e", "Protocol (3, 1) with secp521r1 group",
340 "-e", "Protocol (3, 1) with x25519 group",
341 "-e", "Protocol (3, 2) with secp256r1 group",
342 "-e", "Protocol (3, 2) with secp384r1 group",
343 "-e", "Protocol (3, 2) with secp521r1 group",
344 "-e", "Protocol (3, 2) with x25519 group",
345]
346
347tls12_tests = TestGroup("TLSv1.2 tests", [
348 # Tests that pass as they are.
349 Test("test-aes-gcm-nonces.py"),
350 Test("test-connection-abort.py"),
351 Test("test-conversation.py"),
352 Test("test-cve-2016-2107.py"),
353 Test("test-cve-2016-6309.py"),
354 Test("test-dhe-rsa-key-exchange.py"),
355 Test("test-dhe-rsa-key-exchange-with-bad-messages.py"),
356 Test("test-early-application-data.py"),
357 Test("test-empty-extensions.py"),
358 Test("test-extensions.py"),
359 Test("test-fuzzed-MAC.py"),
360 Test("test-fuzzed-ciphertext.py"),
361 Test("test-fuzzed-finished.py"),
362 Test("test-fuzzed-padding.py"),
363 Test("test-fuzzed-plaintext.py"), # fails once in a while
364 Test("test-hello-request-by-client.py"),
365 Test("test-invalid-cipher-suites.py"),
366 Test("test-invalid-content-type.py"),
367 Test("test-invalid-session-id.py"),
368 Test("test-invalid-version.py"),
369 Test("test-lucky13.py"),
370 Test("test-message-skipping.py"),
371 Test("test-no-heartbeat.py"),
372 Test("test-record-layer-fragmentation.py"),
373 Test("test-sessionID-resumption.py"),
374 Test("test-sslv2-connection.py"),
375 Test("test-truncating-of-finished.py"),
376 Test("test-truncating-of-kRSA-client-key-exchange.py"),
377 Test("test-unsupported-curve-fallback.py"),
378 Test("test-version-numbers.py"),
379 Test("test-zero-length-data.py"),
380
381 # Tests that need tweaking for unsupported features and ciphers.
382 Test(
383 "test-atypical-padding.py", [
384 "-e", "sanity - encrypt then MAC",
385 "-e", "2^14 bytes of AppData with 256 bytes of padding (SHA1 + Encrypt then MAC)",
386 ]
387 ),
388 Test(
389 "test-dhe-rsa-key-exchange-signatures.py", [
390 "-e", "TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA sha224 signature",
391 "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 sha224 signature",
392 "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA sha224 signature",
393 "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 sha224 signature",
394 "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA sha224 signature",
395 ]
396 ),
397 Test("test-dhe-key-share-random.py", tls12_exclude_legacy_protocols),
398 Test("test-export-ciphers-rejected.py", ["--min-ver", "TLSv1.2"]),
399 Test(
400 "test-downgrade-protection.py",
401 tls12_args = ["--server-max-protocol", "TLSv1.2"],
402 tls13_args = [
403 "--server-max-protocol", "TLSv1.3",
404 "-e", "TLS 1.3 downgrade check for Protocol (3, 1)",
405 "-e", "TLS 1.3 downgrade check for Protocol (3, 2)",
406 ]
407 ),
408 Test(
409 "test-fallback-scsv.py",
410 tls13_args = [
411 "--tls-1.3",
412 "-e", "FALLBACK - hello TLSv1.1 - pos 0",
413 "-e", "FALLBACK - hello TLSv1.1 - pos 1",
414 "-e", "FALLBACK - hello TLSv1.1 - pos 2",
415 "-e", "FALLBACK - record TLSv1.1 hello TLSv1.1 - pos 0",
416 "-e", "FALLBACK - record TLSv1.1 hello TLSv1.1 - pos 1",
417 "-e", "FALLBACK - record TLSv1.1 hello TLSv1.1 - pos 2",
418 "-e", "record TLSv1.1 hello TLSv1.1",
419 "-e", "sanity - TLSv1.1",
420 ]
421 ),
422
423 Test("test-invalid-compression-methods.py", [
424 "-x", "invalid compression methods",
425 "-X", substitute_alert("illegal_parameter", "decode_error"),
426 "-x", "only deflate compression method",
427 "-X", substitute_alert("illegal_parameter", "decode_error"),
428 ]),
429
430 # Skip extended_master_secret test. Since we don't support this
431 # extension, we don't notice that it was dropped.
432 Test("test-renegotiation-changed-clienthello.py", [
433 "-e", "drop extended_master_secret in renegotiation",
434 ]),
435
436 # Without --sig-algs-drop-ok, two tests fail since we do not currently
437 # implement the signature_algorithms_cert extension (although we MUST).
438 Test("test-sig-algs-renegotiation-resumption.py", ["--sig-algs-drop-ok"]),
439
440 Test("test-serverhello-random.py", args = tls12_exclude_legacy_protocols),
441
442 Test("test-chacha20.py", [ "-e", "Chacha20 in TLS1.1" ]),
443])
444
445tls12_slow_tests = TestGroup("slow TLSv1.2 tests", [
446 Test("test-cve-2016-7054.py"),
447 Test("test-dhe-no-shared-secret-padding.py", tls12_exclude_legacy_protocols),
448 Test("test-ecdhe-padded-shared-secret.py", tls12_exclude_legacy_protocols),
449 Test("test-ecdhe-rsa-key-share-random.py", tls12_exclude_legacy_protocols),
450 # Start at extension number 58 to avoid QUIC transport parameters (57)
451 Test("test-large-hello.py", [ "-m", "58" ]),
452])
453
454tls12_failing_tests = TestGroup("failing TLSv1.2 tests", [
455 # no shared cipher
456 Test("test-aesccm.py"),
457 # need server to set up alpn
458 Test("test-alpn-negotiation.py"),
459 # Failing on TLS_RSA_WITH_AES_128_CBC_SHA because server does not support it.
460 Test("test-bleichenbacher-timing-pregenerate.py"),
461 # many tests fail due to unexpected server_name extension
462 Test("test-bleichenbacher-workaround.py"),
463
464 # need client key and cert plus extra server setup
465 Test("test-certificate-malformed.py"),
466 Test("test-certificate-request.py"),
467 Test("test-certificate-verify-malformed-sig.py"),
468 Test("test-certificate-verify-malformed.py"),
469 Test("test-certificate-verify.py"),
470 Test("test-ecdsa-in-certificate-verify.py"),
471 Test("test-eddsa-in-certificate-verify.py"),
472 Test("test-renegotiation-disabled-client-cert.py"),
473 Test("test-rsa-pss-sigs-on-certificate-verify.py"),
474 Test("test-rsa-sigs-on-certificate-verify.py"),
475
476 # test doesn't expect session ticket
477 Test("test-client-compatibility.py"),
478 # abrupt closure
479 Test("test-client-hello-max-size.py"),
480 # unknown signature algorithms
481 Test("test-clienthello-md5.py"),
482
483 # Tests expect an illegal_parameter or a decode_error alert. Should be
484 # added to ssl3_get_client_key_exchange on kex function failure.
485 Test("test-ecdhe-rsa-key-exchange-with-bad-messages.py"),
486
487 # We send a handshake_failure due to no shared ciphers while the
488 # test expects to succeed.
489 Test("test-ecdhe-rsa-key-exchange.py"),
490
491 # no shared cipher
492 Test("test-ecdsa-sig-flexibility.py"),
493
494 # Tests expect SH but we send unexpected_message or handshake_failure
495 # 'Application data inside Client Hello'
496 # 'Application data inside Client Key Exchange'
497 # 'Application data inside Finished'
498 Test("test-interleaved-application-data-and-fragmented-handshakes-in-renegotiation.py"),
499 # Tests expect SH but we send handshake_failure
500 # 'Application data before Change Cipher Spec'
501 # 'Application data before Client Key Exchange'
502 # 'Application data before Finished'
503 Test("test-interleaved-application-data-in-renegotiation.py"),
504
505 # broken test script
506 # TypeError: '<' not supported between instances of 'int' and 'NoneType'
507 Test("test-invalid-client-hello-w-record-overflow.py"),
508
509 # Lots of failures. abrupt closure
510 Test("test-invalid-client-hello.py"),
511
512 # abrupt closure
513 # 'encrypted premaster set to all zero (n)' n in 256 384 512
514 Test("test-invalid-rsa-key-exchange-messages.py"),
515
516 # test expects illegal_parameter, we send unrecognized_name (which seems
517 # correct according to rfc 6066?)
518 Test("test-invalid-server-name-extension-resumption.py"),
519 # let through some server names without sending an alert
520 # again illegal_parameter vs unrecognized_name
521 Test("test-invalid-server-name-extension.py"),
522
523 # 14 pass
524 # 7 fail
525 # 'n extensions', n in 4095, 4096, 4097, 8191, 8192, 8193, 16383,
526 Test("test-large-number-of-extensions.py"),
527
528 # 4 failures:
529 # 'insecure (legacy) renegotiation with GET after 2nd handshake'
530 # 'insecure (legacy) renegotiation with incomplete GET'
531 # 'secure renegotiation with GET after 2nd handshake'
532 # 'secure renegotiation with incomplete GET'
533 Test("test-legacy-renegotiation.py"),
534
535 # 1 failure (timeout): we don't send the unexpected_message alert
536 # 'duplicate change cipher spec after Finished'
537 Test("test-message-duplication.py"),
538
539 # server should send status_request
540 Test("test-ocsp-stapling.py"),
541
542 # unexpected closure
543 Test("test-openssl-3712.py"),
544
545 # failed: 3 (expect an alert, we send AD)
546 # 'try insecure (legacy) renegotiation with incomplete GET'
547 # 'try secure renegotiation with GET after 2nd CH'
548 # 'try secure renegotiation with incomplete GET'
549 Test("test-renegotiation-disabled.py"),
550
551 # 'resumption of safe session with NULL cipher'
552 # 'resumption with cipher from old CH but not selected by server'
553 Test("test-resumption-with-wrong-ciphers.py"),
554
555 # 5 failures:
556 # 'empty sigalgs'
557 # 'only undefined sigalgs'
558 # 'rsa_pss_pss_sha256 only'
559 # 'rsa_pss_pss_sha384 only'
560 # 'rsa_pss_pss_sha512 only'
561 Test("test-sig-algs.py"),
562
563 # 13 failures:
564 # 'duplicated n non-rsa schemes' for n in 202 2342 8119 23741 32744
565 # 'empty list of signature methods'
566 # 'tolerance n RSA or ECDSA methods' for n in 215 2355 8132 23754
567 # 'tolerance 32758 methods with sig_alg_cert'
568 # 'tolerance max 32744 number of methods with sig_alg_cert'
569 # 'tolerance max (32760) number of methods'
570 Test("test-signature-algorithms.py"),
571
572 # times out
573 Test("test-ssl-death-alert.py"),
574
575 # 17 pass, 13 fail. padding and truncation
576 Test("test-truncating-of-client-hello.py"),
577
578 # x448 tests need disabling plus x25519 corner cases need sorting out
579 Test("test-x25519.py"),
580
581 # Needs TLS 1.0 or 1.1
582 Test("test-TLSv1_2-rejected-without-TLSv1_2.py"),
583])
584
585tls12_unsupported_tests = TestGroup("TLSv1.2 for unsupported features", [
586 # protocol_version
587 Test("test-SSLv3-padding.py"),
588 # we don't do RSA key exchanges
589 Test("test-bleichenbacher-timing.py"),
590 # no encrypt-then-mac
591 Test("test-encrypt-then-mac-renegotiation.py"),
592 Test("test-encrypt-then-mac.py"),
593 # no EME support
594 Test("test-extended-master-secret-extension-with-client-cert.py"),
595 Test("test-extended-master-secret-extension.py"),
596 # no ffdhe
597 Test("test-ffdhe-expected-params.py"),
598 Test("test-ffdhe-negotiation.py"),
599 # record_size_limit/max_fragment_length extension (RFC 8449)
600 Test("test-record-size-limit.py"),
601 # expects the server to send the heartbeat extension
602 Test("test-heartbeat.py"),
603 # needs an echo server
604 Test("test-lengths.py"),
605])
606
607# These tests take a ton of time to fail against an 1.3 server,
608# so don't run them against 1.3 pending further investigation.
609legacy_tests = TestGroup("Legacy protocol tests", [
610 Test("test-sslv2-force-cipher-3des.py"),
611 Test("test-sslv2-force-cipher-non3des.py"),
612 Test("test-sslv2-force-cipher.py"),
613 Test("test-sslv2-force-export-cipher.py"),
614 Test("test-sslv2hello-protocol.py"),
615])
616
617all_groups = [
618 tls13_tests,
619 tls13_slow_tests,
620 tls13_extra_cert_tests,
621 tls13_failing_tests,
622 tls13_slow_failing_tests,
623 tls13_unsupported_tests,
624 tls12_tests,
625 tls12_slow_tests,
626 tls12_failing_tests,
627 tls12_unsupported_tests,
628 legacy_tests,
629]
630
631failing_groups = [
632 tls13_failing_tests,
633 tls13_slow_failing_tests,
634 tls12_failing_tests,
635]
636
637class TestRunner:
638 """ Runs the given tests troups against a server and displays stats. """
639
640 def __init__(
641 self, timing=False, verbose=False, host="localhost", port=4433,
642 use_tls1_3=True, dry_run=False, tests=[], scriptdir=tlsfuzzer_scriptdir,
643 ):
644 self.tests = []
645
646 self.dryrun = dry_run
647 self.use_tls1_3 = use_tls1_3
648 self.host = host
649 self.port = str(port)
650 self.scriptdir = scriptdir
651
652 self.stats = []
653 self.failed = []
654 self.missing = []
655
656 self.timing = timing
657 self.verbose = verbose
658
659 def add(self, title="tests", tests=[]):
660 # tests.sort(key=lambda test: test.name)
661 self.tests.append(TestGroup(title, tests))
662
663 def add_group(self, group):
664 self.tests.append(group)
665
666 def run_script(self, test):
667 script = test.name
668 args = ["-h"] + [self.host] + ["-p"] + [self.port] + test.args(self.use_tls1_3)
669
670 if self.dryrun:
671 if not self.verbose:
672 args = []
673 print(script , end=' ' if args else '')
674 print(' '.join([f"\"{arg}\"" for arg in args]))
675 return
676
677 if self.verbose:
678 print(script)
679 else:
680 print(f"{script[:68]:<72}", end=" ", flush=True)
681 start = timer()
682 scriptpath = os.path.join(self.scriptdir, script)
683 if not os.path.exists(scriptpath):
684 self.missing.append(script)
685 print("MISSING")
686 return
687 test = subprocess.run(
688 ["python3", scriptpath] + args,
689 capture_output=not self.verbose,
690 text=True,
691 )
692 end = timer()
693 self.stats.append((script, end - start))
694 if test.returncode == 0:
695 print("OK")
696 return
697 print("FAILED")
698 self.failed.append(script)
699
700 if self.verbose:
701 return
702
703 print('\n'.join(test.stdout.split("Test end\n", 1)[1:]), end="")
704
705 def run(self):
706 for group in self:
707 print(f"Running {group.title} ...")
708 for test in group:
709 self.run_script(test)
710 return not self.failed
711
712 def __iter__(self):
713 return iter(self.tests)
714
715 def __del__(self):
716 if self.timing and self.stats:
717 total = 0.0
718 for (script, time) in self.stats:
719 print(f"{round(time, 2):6.2f} {script}")
720 total += time
721 print(f"{round(total, 2):6.2f} total")
722
723 if self.failed:
724 print("Failed tests:")
725 print('\n'.join(self.failed))
726
727 if self.missing:
728 print("Missing tests (outdated package?):")
729 print('\n'.join(self.missing))
730
731class TlsServer:
732 """ Spawns an s_server listening on localhost:port if necessary. """
733
734 def __init__(self, host="localhost", port=4433):
735 self.spawn = True
736 # Check whether a server is already listening on localhost:port
737 self.spawn = subprocess.run(
738 ["nc", "-c", "-z", "-T", "noverify", host, str(port)],
739 stderr=subprocess.DEVNULL,
740 ).returncode != 0
741
742 if self.spawn:
743 self.server = subprocess.Popen(
744 [
745 "openssl",
746 "s_server",
747 "-accept",
748 str(port),
749 "-groups",
750 "X25519:P-256:P-521:P-384",
751 "-key",
752 "localhost.key",
753 "-cert",
754 "localhost.crt",
755 "-www",
756 ],
757 stdout=subprocess.DEVNULL,
758 stderr=subprocess.PIPE,
759 text=True,
760 )
761
762 # Check whether the server talks TLSv1.3
763 self.has_tls1_3 = True or subprocess.run(
764 [
765 "nc",
766 "-c",
767 "-z",
768 "-T",
769 "noverify",
770 "-T",
771 "protocols=TLSv1.3",
772 "localhost",
773 str(port),
774 ],
775 stderr=subprocess.DEVNULL,
776 ).returncode == 0
777
778 self.check()
779
780 def check(self):
781 if self.spawn and self.server.poll() is not None:
782 print(self.server.stderr.read())
783 raise RuntimeError(
784 f"openssl s_server died. Return code: {self.server.returncode}."
785 )
786 if self.spawn:
787 self.server.stderr.detach()
788
789 def __del__(self):
790 if self.spawn:
791 self.server.terminate()
792
793# Extract the arguments we pass to script
794def defaultargs(script, has_tls1_3):
795 return next(
796 (test for group in all_groups for test in group if test.name == script),
797 Test()
798 ).args(has_tls1_3)
799
800def list_or_missing(missing=True):
801 tests = [test.name for group in all_groups for test in group]
802
803 if missing:
804 scripts = {
805 f for f in os.listdir(tlsfuzzer_scriptdir) if f != "__pycache__"
806 }
807 missing = scripts - set(tests)
808 if missing:
809 print('\n'.join(sorted(missing)))
810 exit(0)
811
812 tests.sort()
813 print('\n'.join(tests))
814 exit(0)
815
816def usage():
817 print("Usage: python3 tlsfuzzer.py [-flmnstv] [-p port] [script [test...]]")
818 print(" --help help")
819 print(" -f run failing tests")
820 print(" -l list tests")
821 print(" -m list new tests after package update")
822 print(" -n do not run tests, but list the ones that would be run")
823 print(" -p port connect to this port - defaults to 4433")
824 print(" -s run slow tests")
825 print(" -t show timing stats at end")
826 print(" -v verbose output")
827 exit(0)
828
829def main():
830 failing = False
831 list = False
832 missing = False
833 dryrun = False
834 host = "localhost"
835 port = 4433
836 slow = False
837 timing = False
838 verbose = False
839
840 argv = sys.argv[1:]
841 opts, args = getopt.getopt(argv, "fh:lmnp:stv", ["help"])
842 for opt, arg in opts:
843 if opt == '--help':
844 usage()
845 elif opt == '-f':
846 failing = True
847 elif opt == '-h':
848 host = arg
849 elif opt == '-l':
850 list = True
851 elif opt == '-m':
852 missing = True
853 elif opt == '-n':
854 dryrun = True
855 elif opt == '-p':
856 port = int(arg)
857 elif opt == '-s':
858 slow = True
859 elif opt == '-t':
860 timing = True
861 elif opt == '-v':
862 verbose = True
863 else:
864 raise ValueError(f"Unknown option: {opt}")
865
866 if not os.path.exists(tlsfuzzer_scriptdir):
867 print("package py3-tlsfuzzer is required for this regress")
868 exit(1)
869
870 if list and failing:
871 failing = [test.name for group in failing_groups for test in group]
872 failing.sort()
873 print('\n'.join(failing))
874 exit(0)
875
876 if list or missing:
877 list_or_missing(missing)
878
879 tls_server = TlsServer(host, port)
880
881 tests = TestRunner(timing, verbose, host, port, tls_server.has_tls1_3, dryrun)
882
883 if args:
884 (dir, script) = os.path.split(args[0])
885 if dir and not dir == '.':
886 tests.scriptdir = dir
887
888 testargs = defaultargs(script, tls_server.has_tls1_3)
889
890 tests.verbose = True
891 tests.add("test from command line", [Test(script, testargs + args[1:])])
892
893 exit(not tests.run())
894
895 if failing:
896 if tls_server.has_tls1_3:
897 tests.add_group(tls13_failing_tests)
898 if slow:
899 tests.add_group(tls13_slow_failing_tests)
900 tests.add_group(tls12_failing_tests)
901
902 if tls_server.has_tls1_3:
903 tests.add_group(tls13_tests)
904 if slow:
905 tests.add_group(tls13_slow_tests)
906 else:
907 tests.add_group(legacy_tests)
908
909 tests.add_group(tls12_tests)
910 if slow:
911 tests.add_group(tls12_slow_tests)
912
913 success = tests.run()
914 del tests
915
916 if not success:
917 print("FAILED")
918 exit(1)
919
920if __name__ == "__main__":
921 main()
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2025-12-28 20:11:01 +0000