summaryrefslogtreecommitdiff
path: root/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py
diff options
context:
space:
mode:
Diffstat (limited to 'src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py')
-rw-r--r--src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py935
1 files changed, 0 insertions, 935 deletions
diff --git a/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py b/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py
deleted file mode 100644
index 91aedad165..0000000000
--- a/src/regress/lib/libssl/tlsfuzzer/tlsfuzzer.py
+++ /dev/null
@@ -1,935 +0,0 @@
1# $OpenBSD: tlsfuzzer.py,v 1.56 2024/09/18 19:12:37 tb Exp $
2#
3# Copyright (c) 2020 Theo Buehler <tb@openbsd.org>
4#
5# Permission to use, copy, modify, and distribute this software for any
6# purpose with or without fee is hereby granted, provided that the above
7# copyright notice and this permission notice appear in all copies.
8#
9# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16
17import getopt
18import os
19import subprocess
20import sys
21from timeit import default_timer as timer
22
23tlsfuzzer_scriptdir = "/usr/local/share/tlsfuzzer/scripts/"
24
25class Test:
26 """
27 Represents a tlsfuzzer test script.
28 name: the script's name
29 args: arguments to feed to the script
30 tls12_args: override args for a TLSv1.2 server
31 tls13_args: override args for a TLSv1.3 server
32
33 XXX Add client cert support.
34 """
35 def __init__(self, name="", args=[], tls12_args=[], tls13_args=[]):
36 self.name = name
37 self.tls12_args = args
38 self.tls13_args = args
39 if tls12_args:
40 self.tls12_args = tls12_args
41 if tls13_args:
42 self.tls13_args = tls13_args
43
44 def args(self, has_tls1_3: True):
45 if has_tls1_3:
46 return self.tls13_args
47 else:
48 return self.tls12_args
49
50 def __repr__(self):
51 return "<Test: %s tls12_args: %s tls13_args: %s>" % (
52 self.name, self.tls12_args, self.tls13_args
53 )
54
55class TestGroup:
56 """ A group of Test objects to be run by TestRunner."""
57 def __init__(self, title="Tests", tests=[]):
58 self.title = title
59 self.tests = tests
60
61 def __iter__(self):
62 return iter(self.tests)
63
64# argument to pass to several tests
65tls13_unsupported_ciphers = [
66 "-e", "TLS 1.3 with ffdhe2048",
67 "-e", "TLS 1.3 with ffdhe3072",
68 "-e", "TLS 1.3 with x448",
69]
70
71def substitute_alert(want, got):
72 return f"Expected alert description \"{want}\" " \
73 + f"does not match received \"{got}\""
74
75# test-tls13-finished.py has 70 failing tests that expect a "decode_error"
76# instead of the "decrypt_error" sent by tls13_server_finished_recv().
77# Both alerts appear to be reasonable in this context, so work around this
78# in the test instead of the library.
79def generate_test_tls13_finished_args():
80 assertion = substitute_alert("decode_error", "decrypt_error");
81 paddings = [
82 ("TLS_AES_128_GCM_SHA256", 0, 1),
83 ("TLS_AES_128_GCM_SHA256", 0, 2),
84 ("TLS_AES_128_GCM_SHA256", 0, 4),
85 ("TLS_AES_128_GCM_SHA256", 0, 8),
86 ("TLS_AES_128_GCM_SHA256", 0, 16),
87 ("TLS_AES_128_GCM_SHA256", 0, 32),
88 ("TLS_AES_128_GCM_SHA256", 0, 48),
89 ("TLS_AES_128_GCM_SHA256", 0, 2**14-4-32),
90 ("TLS_AES_128_GCM_SHA256", 0, 0x20000),
91 ("TLS_AES_128_GCM_SHA256", 0, 0x30000),
92 ("TLS_AES_128_GCM_SHA256", 1, 0),
93 ("TLS_AES_128_GCM_SHA256", 2, 0),
94 ("TLS_AES_128_GCM_SHA256", 4, 0),
95 ("TLS_AES_128_GCM_SHA256", 8, 0),
96 ("TLS_AES_128_GCM_SHA256", 16, 0),
97 ("TLS_AES_128_GCM_SHA256", 32, 0),
98 ("TLS_AES_128_GCM_SHA256", 48, 0),
99 ("TLS_AES_128_GCM_SHA256", 2**14-4-32, 0),
100 ("TLS_AES_128_GCM_SHA256", 12, 0),
101 ("TLS_AES_128_GCM_SHA256", 1, 1),
102 ("TLS_AES_128_GCM_SHA256", 8, 8),
103 ("TLS_AES_256_GCM_SHA384", 0, 1),
104 ("TLS_AES_256_GCM_SHA384", 0, 2),
105 ("TLS_AES_256_GCM_SHA384", 0, 4),
106 ("TLS_AES_256_GCM_SHA384", 0, 8),
107 ("TLS_AES_256_GCM_SHA384", 0, 16),
108 ("TLS_AES_256_GCM_SHA384", 0, 32),
109 ("TLS_AES_256_GCM_SHA384", 0, 48),
110 ("TLS_AES_256_GCM_SHA384", 0, 2**14-4-48),
111 ("TLS_AES_256_GCM_SHA384", 0, 0x20000),
112 ("TLS_AES_256_GCM_SHA384", 0, 0x30000),
113 ("TLS_AES_256_GCM_SHA384", 0, 12),
114 ("TLS_AES_256_GCM_SHA384", 1, 0),
115 ("TLS_AES_256_GCM_SHA384", 2, 0),
116 ("TLS_AES_256_GCM_SHA384", 4, 0),
117 ("TLS_AES_256_GCM_SHA384", 8, 0),
118 ("TLS_AES_256_GCM_SHA384", 16, 0),
119 ("TLS_AES_256_GCM_SHA384", 32, 0),
120 ("TLS_AES_256_GCM_SHA384", 48, 0),
121 ("TLS_AES_256_GCM_SHA384", 2**14-4-48, 0),
122 ("TLS_AES_256_GCM_SHA384", 1, 1),
123 ("TLS_AES_256_GCM_SHA384", 8, 8),
124 ]
125 truncations = [
126 ("TLS_AES_128_GCM_SHA256", 0, -1),
127 ("TLS_AES_128_GCM_SHA256", 0, -2),
128 ("TLS_AES_128_GCM_SHA256", 0, -4),
129 ("TLS_AES_128_GCM_SHA256", 0, -8),
130 ("TLS_AES_128_GCM_SHA256", 0, -16),
131 ("TLS_AES_128_GCM_SHA256", 0, -32),
132 ("TLS_AES_128_GCM_SHA256", 0, 12),
133 ("TLS_AES_128_GCM_SHA256", 1, None),
134 ("TLS_AES_128_GCM_SHA256", 2, None),
135 ("TLS_AES_128_GCM_SHA256", 4, None),
136 ("TLS_AES_128_GCM_SHA256", 8, None),
137 ("TLS_AES_128_GCM_SHA256", 16, None),
138 ("TLS_AES_128_GCM_SHA256", 32, None),
139 ("TLS_AES_256_GCM_SHA384", 0, -1),
140 ("TLS_AES_256_GCM_SHA384", 0, -2),
141 ("TLS_AES_256_GCM_SHA384", 0, -4),
142 ("TLS_AES_256_GCM_SHA384", 0, -8),
143 ("TLS_AES_256_GCM_SHA384", 0, -16),
144 ("TLS_AES_256_GCM_SHA384", 0, -32),
145 ("TLS_AES_256_GCM_SHA384", 0, 12),
146 ("TLS_AES_256_GCM_SHA384", 1, None),
147 ("TLS_AES_256_GCM_SHA384", 2, None),
148 ("TLS_AES_256_GCM_SHA384", 4, None),
149 ("TLS_AES_256_GCM_SHA384", 8, None),
150 ("TLS_AES_256_GCM_SHA384", 16, None),
151 ("TLS_AES_256_GCM_SHA384", 32, None),
152 ]
153
154 args = [
155 "-x", "empty - cipher TLS_AES_128_GCM_SHA256", "-X", assertion,
156 "-x", "empty - cipher TLS_AES_256_GCM_SHA384", "-X", assertion,
157 ]
158 padding_fmt = "padding - cipher %s, pad_byte 0, pad_left %d, pad_right %d"
159 for padding in paddings:
160 args += ["-x", padding_fmt % padding, "-X", assertion]
161 truncation_fmt = "truncation - cipher %s, start %d, end %s"
162 for truncation in truncations:
163 args += ["-x", truncation_fmt % truncation, "-X", assertion]
164 return args
165
166tls13_tests = TestGroup("TLSv1.3 tests", [
167 Test("test-tls13-ccs.py"),
168 Test("test-tls13-conversation.py"),
169 Test("test-tls13-count-tickets.py"),
170 Test("test-tls13-empty-alert.py"),
171 Test("test-tls13-finished.py", generate_test_tls13_finished_args()),
172 Test("test-tls13-finished-plaintext.py"),
173 Test("test-tls13-hrr.py"),
174 Test("test-tls13-keyshare-omitted.py"),
175 Test("test-tls13-legacy-version.py"),
176 Test("test-tls13-nociphers.py"),
177 Test("test-tls13-record-padding.py"),
178 # Exclude QUIC transport parameters
179 Test("test-tls13-shuffled-extentions.py", [ "--exc", "57" ]),
180 Test("test-tls13-zero-content-type.py"),
181
182 # The skipped tests fail due to a bug in BIO_gets() which masks the retry
183 # signalled from an SSL_read() failure. Testing with httpd(8) shows we're
184 # handling these corner cases correctly since tls13_record_layer.c -r1.47.
185 Test("test-tls13-zero-length-data.py", [
186 "-e", "zero-length app data",
187 "-e", "zero-length app data with large padding",
188 "-e", "zero-length app data with padding",
189 ]),
190
191 # We don't currently handle NSTs
192 Test("test-tls13-connection-abort.py", ["-e", "After NewSessionTicket"]),
193])
194
195# Tests that take a lot of time (> ~30s on an x280)
196tls13_slow_tests = TestGroup("slow TLSv1.3 tests", [
197 # XXX: Investigate the occasional message
198 # "Got shared secret with 1 most significant bytes equal to zero."
199 Test("test-tls13-dhe-shared-secret-padding.py", tls13_unsupported_ciphers),
200
201 Test("test-tls13-invalid-ciphers.py"),
202 Test("test-tls13-serverhello-random.py", tls13_unsupported_ciphers),
203
204 # Mark two tests cases as xfail for now. The tests expect an arguably
205 # correct decode_error while we send a decrypt_error (like fizz/boring).
206 Test("test-tls13-record-layer-limits.py", [
207 "-x", "max size payload (2**14) of Finished msg, with 16348 bytes of left padding, cipher TLS_AES_128_GCM_SHA256",
208 "-X", substitute_alert("decode_error", "decrypt_error"),
209 "-x", "max size payload (2**14) of Finished msg, with 16348 bytes of left padding, cipher TLS_CHACHA20_POLY1305_SHA256",
210 "-X", substitute_alert("decode_error", "decrypt_error"),
211 ]),
212 # We don't accept an empty ECPF extension since it must advertise the
213 # uncompressed point format. Exclude this extension type from the test.
214 Test(
215 "test-tls13-large-number-of-extensions.py",
216 tls13_args = ["--exc", "11"],
217 ),
218])
219
220tls13_extra_cert_tests = TestGroup("TLSv1.3 certificate tests", [
221 # need to set up client certs to run these
222 Test("test-tls13-certificate-request.py"),
223 Test("test-tls13-certificate-verify.py"),
224 Test("test-tls13-ecdsa-in-certificate-verify.py"),
225 Test("test-tls13-eddsa-in-certificate-verify.py"),
226
227 # Test expects the server to have installed three certificates:
228 # with P-256, P-384 and P-521 curve. Also SHA1+ECDSA is verified
229 # to not work.
230 Test("test-tls13-ecdsa-support.py"),
231])
232
233tls13_failing_tests = TestGroup("failing TLSv1.3 tests", [
234 # Some tests fail because we fail later than the scripts expect us to.
235 # With X25519, we accept weak peer public keys and fail when we actually
236 # compute the keyshare. Other tests seem to indicate that we could be
237 # stricter about what keyshares we accept.
238 Test("test-tls13-crfg-curves.py", [
239 '-e', 'all zero x448 key share',
240 '-e', 'empty x448 key share',
241 '-e', 'sanity x448 with compression ansiX962_compressed_char2',
242 '-e', 'sanity x448 with compression ansiX962_compressed_prime',
243 '-e', 'sanity x448 with compression uncompressed',
244 '-e', 'too big x448 key share',
245 '-e', 'too small x448 key share',
246 '-e', 'x448 key share of "1"',
247 ]),
248 Test("test-tls13-ecdhe-curves.py", [
249 '-e', 'sanity - x448',
250 '-e', 'x448 - key share from other curve',
251 '-e', 'x448 - point at infinity',
252 '-e', 'x448 - right 0-padded key_share',
253 '-e', 'x448 - right-truncated key_share',
254 ]),
255
256 # The test sends records with protocol version 0x0300 instead of 0x0303
257 # and currently fails with OpenSSL and LibreSSL for this reason.
258 # We have the logic corresponding to NSS's fix for CVE-2020-25648
259 # https://hg.mozilla.org/projects/nss/rev/57bbefa793232586d27cee83e74411171e128361
260 # so should not be affected by this issue.
261 Test("test-tls13-multiple-ccs-messages.py"),
262
263 # https://github.com/openssl/openssl/issues/8369
264 Test("test-tls13-obsolete-curves.py"),
265
266 # 3 failing rsa_pss_pss tests
267 Test("test-tls13-rsa-signatures.py"),
268
269 # The failing tests all expect an ri extension. What's up with that?
270 Test("test-tls13-version-negotiation.py"),
271])
272
273tls13_slow_failing_tests = TestGroup("slow, failing TLSv1.3 tests", [
274 # Other test failures bugs in keyshare/tlsext negotiation?
275 Test("test-tls13-unrecognised-groups.py"), # unexpected closure
276
277 # 5 occasional failures:
278 # 'app data split, conversation with KeyUpdate msg'
279 # 'fragmented keyupdate msg'
280 # 'multiple KeyUpdate messages'
281 # 'post-handshake KeyUpdate msg with update_not_request'
282 # 'post-handshake KeyUpdate msg with update_request'
283 Test("test-tls13-keyupdate.py"),
284
285 Test("test-tls13-symetric-ciphers.py"), # unexpected message from peer
286
287 # 6 tests fail: 'rsa_pkcs1_{md5,sha{1,224,256,384,512}} signature'
288 # We send server hello, but the test expects handshake_failure
289 Test("test-tls13-pkcs-signature.py"),
290 # 8 tests fail: 'tls13 signature rsa_pss_{pss,rsae}_sha{256,384,512}
291 Test("test-tls13-rsapss-signatures.py"),
292])
293
294tls13_unsupported_tests = TestGroup("TLSv1.3 tests for unsupported features", [
295 # Tests for features we don't support
296 Test("test-tls13-0rtt-garbage.py"),
297 Test("test-tls13-ffdhe-groups.py"),
298 Test("test-tls13-ffdhe-sanity.py"),
299 Test("test-tls13-psk_dhe_ke.py"),
300 Test("test-tls13-psk_ke.py"),
301
302 # need server to react to HTTP GET for /keyupdate
303 Test("test-tls13-keyupdate-from-server.py"),
304
305 # needs an echo server
306 Test("test-tls13-lengths.py"),
307
308 # Weird test: tests servers that don't support 1.3
309 Test("test-tls13-non-support.py"),
310
311 # broken test script
312 # UnboundLocalError: local variable 'cert' referenced before assignment
313 Test("test-tls13-post-handshake-auth.py"),
314
315 # ExpectNewSessionTicket
316 Test("test-tls13-session-resumption.py"),
317
318 # Server must be configured to support only rsa_pss_rsae_sha512
319 Test("test-tls13-signature-algorithms.py"),
320])
321
322tls12_exclude_legacy_protocols = [
323 # all these have BIO_read timeouts against TLSv1.3
324 "-e", "Protocol (3, 0)",
325 "-e", "Protocol (3, 1)",
326 "-e", "Protocol (3, 2)",
327 "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello",
328 # the following only fail with TLSv1.3
329 "-e", "Protocol (3, 1) in SSLv2 compatible ClientHello",
330 "-e", "Protocol (3, 2) in SSLv2 compatible ClientHello",
331 "-e", "Protocol (3, 3) in SSLv2 compatible ClientHello",
332 "-e", "Protocol (3, 1) with x448 group",
333 "-e", "Protocol (3, 2) with x448 group",
334 "-e", "Protocol (3, 3) with x448 group",
335 # These don't work without TLSv1.0 and TLSv1.1
336 "-e", "Protocol (3, 1) with secp256r1 group",
337 "-e", "Protocol (3, 1) with secp384r1 group",
338 "-e", "Protocol (3, 1) with secp521r1 group",
339 "-e", "Protocol (3, 1) with x25519 group",
340 "-e", "Protocol (3, 2) with secp256r1 group",
341 "-e", "Protocol (3, 2) with secp384r1 group",
342 "-e", "Protocol (3, 2) with secp521r1 group",
343 "-e", "Protocol (3, 2) with x25519 group",
344]
345
346tls12_tests = TestGroup("TLSv1.2 tests", [
347 # Tests that pass as they are.
348 Test("test-aes-gcm-nonces.py"),
349 Test("test-connection-abort.py"),
350 Test("test-conversation.py"),
351 Test("test-cve-2016-2107.py"),
352 Test("test-cve-2016-6309.py"),
353 Test("test-dhe-rsa-key-exchange.py"),
354 Test("test-early-application-data.py"),
355 Test("test-empty-extensions.py"),
356 Test("test-extensions.py"),
357 Test("test-fuzzed-MAC.py"),
358 Test("test-fuzzed-ciphertext.py"),
359 Test("test-fuzzed-finished.py"),
360 Test("test-fuzzed-padding.py"),
361 Test("test-fuzzed-plaintext.py"), # fails once in a while
362 Test("test-hello-request-by-client.py"),
363 Test("test-invalid-cipher-suites.py"),
364 Test("test-invalid-content-type.py"),
365 Test("test-invalid-session-id.py"),
366 Test("test-invalid-version.py"),
367 Test("test-large-number-of-extensions.py"),
368 Test("test-lucky13.py"),
369 Test("test-message-skipping.py"),
370 Test("test-no-heartbeat.py"),
371 Test("test-record-layer-fragmentation.py"),
372 Test("test-sslv2-connection.py"),
373 Test("test-truncating-of-finished.py"),
374 Test("test-truncating-of-kRSA-client-key-exchange.py"),
375 Test("test-unsupported-curve-fallback.py"),
376 Test("test-version-numbers.py"),
377 Test("test-zero-length-data.py"),
378
379 # Tests that need tweaking for unsupported features and ciphers.
380 Test(
381 "test-atypical-padding.py", [
382 "-e", "sanity - encrypt then MAC",
383 "-e", "2^14 bytes of AppData with 256 bytes of padding (SHA1 + Encrypt then MAC)",
384 ]
385 ),
386 Test(
387 "test-ccs.py", [
388 "-x", "two bytes long CCS",
389 "-X", substitute_alert("unexpected_message", "decode_error"),
390 ]
391 ),
392 Test(
393 "test-dhe-rsa-key-exchange-signatures.py", [
394 "-e", "TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA sha224 signature",
395 "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 sha224 signature",
396 "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA sha224 signature",
397 "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 sha224 signature",
398 "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA sha224 signature",
399 ]
400 ),
401 Test("test-dhe-rsa-key-exchange-with-bad-messages.py", [
402 "-x", "invalid dh_Yc value - missing",
403 "-X", substitute_alert("decode_error", "illegal_parameter"),
404 ]),
405 Test("test-dhe-key-share-random.py", tls12_exclude_legacy_protocols),
406 Test("test-export-ciphers-rejected.py", ["--min-ver", "TLSv1.2"]),
407 Test(
408 "test-downgrade-protection.py",
409 tls12_args = ["--server-max-protocol", "TLSv1.2"],
410 tls13_args = [
411 "--server-max-protocol", "TLSv1.3",
412 "-e", "TLS 1.3 downgrade check for Protocol (3, 1)",
413 "-e", "TLS 1.3 downgrade check for Protocol (3, 2)",
414 ]
415 ),
416 Test(
417 "test-fallback-scsv.py",
418 tls13_args = [
419 "--tls-1.3",
420 "-e", "FALLBACK - hello TLSv1.1 - pos 0",
421 "-e", "FALLBACK - hello TLSv1.1 - pos 1",
422 "-e", "FALLBACK - hello TLSv1.1 - pos 2",
423 "-e", "FALLBACK - record TLSv1.1 hello TLSv1.1 - pos 0",
424 "-e", "FALLBACK - record TLSv1.1 hello TLSv1.1 - pos 1",
425 "-e", "FALLBACK - record TLSv1.1 hello TLSv1.1 - pos 2",
426 "-e", "record TLSv1.1 hello TLSv1.1",
427 "-e", "sanity - TLSv1.1",
428 ]
429 ),
430
431 Test("test-invalid-compression-methods.py", [
432 "-x", "invalid compression methods",
433 "-X", substitute_alert("illegal_parameter", "decode_error"),
434 "-x", "only deflate compression method",
435 "-X", substitute_alert("illegal_parameter", "decode_error"),
436 ]),
437
438 # Skip extended_master_secret test. Since we don't support this
439 # extension, we don't notice that it was dropped.
440 Test("test-renegotiation-changed-clienthello.py", [
441 "-e", "drop extended_master_secret in renegotiation",
442 ]),
443
444 Test("test-sessionID-resumption.py", [
445 "-x", "Client Hello too long session ID",
446 "-X", substitute_alert("decode_error", "illegal_parameter"),
447 ]),
448
449 # Without --sig-algs-drop-ok, two tests fail since we do not currently
450 # implement the signature_algorithms_cert extension (although we MUST).
451 Test("test-sig-algs-renegotiation-resumption.py", ["--sig-algs-drop-ok"]),
452
453 Test("test-serverhello-random.py", args = tls12_exclude_legacy_protocols),
454
455 Test("test-chacha20.py", [ "-e", "Chacha20 in TLS1.1" ]),
456])
457
458tls12_slow_tests = TestGroup("slow TLSv1.2 tests", [
459 Test("test-cve-2016-7054.py"),
460 Test("test-dhe-no-shared-secret-padding.py", tls12_exclude_legacy_protocols),
461 Test("test-ecdhe-padded-shared-secret.py", tls12_exclude_legacy_protocols),
462 Test("test-ecdhe-rsa-key-share-random.py", tls12_exclude_legacy_protocols),
463 # Start at extension number 58 to avoid QUIC transport parameters (57)
464 Test("test-large-hello.py", [ "-m", "58" ]),
465])
466
467tls12_failing_tests = TestGroup("failing TLSv1.2 tests", [
468 # no shared cipher
469 Test("test-aesccm.py"),
470 # need server to set up alpn
471 Test("test-alpn-negotiation.py"),
472 # Failing on TLS_RSA_WITH_AES_128_CBC_SHA because server does not support it.
473 Test("test-bleichenbacher-timing-pregenerate.py"),
474 # many tests fail due to unexpected server_name extension
475 Test("test-bleichenbacher-workaround.py"),
476
477 # need client key and cert plus extra server setup
478 Test("test-certificate-malformed.py"),
479 Test("test-certificate-request.py"),
480 Test("test-certificate-verify-malformed-sig.py"),
481 Test("test-certificate-verify-malformed.py"),
482 Test("test-certificate-verify.py"),
483 Test("test-ecdsa-in-certificate-verify.py"),
484 Test("test-eddsa-in-certificate-verify.py"),
485 Test("test-renegotiation-disabled-client-cert.py"),
486 Test("test-rsa-pss-sigs-on-certificate-verify.py"),
487 Test("test-rsa-sigs-on-certificate-verify.py"),
488
489 # test doesn't expect session ticket
490 Test("test-client-compatibility.py"),
491 # abrupt closure
492 Test("test-client-hello-max-size.py"),
493 # unknown signature algorithms
494 Test("test-clienthello-md5.py"),
495
496 # Tests expect an illegal_parameter or a decode_error alert. Should be
497 # added to ssl3_get_client_key_exchange on kex function failure.
498 Test("test-ecdhe-rsa-key-exchange-with-bad-messages.py"),
499
500 # We send a handshake_failure due to no shared ciphers while the
501 # test expects to succeed.
502 Test("test-ecdhe-rsa-key-exchange.py"),
503
504 # no shared cipher
505 Test("test-ecdsa-sig-flexibility.py"),
506
507 # Tests expect SH but we send unexpected_message or handshake_failure
508 # 'Application data inside Client Hello'
509 # 'Application data inside Client Key Exchange'
510 # 'Application data inside Finished'
511 Test("test-interleaved-application-data-and-fragmented-handshakes-in-renegotiation.py"),
512 # Tests expect SH but we send handshake_failure
513 # 'Application data before Change Cipher Spec'
514 # 'Application data before Client Key Exchange'
515 # 'Application data before Finished'
516 Test("test-interleaved-application-data-in-renegotiation.py"),
517
518 # broken test script
519 # TypeError: '<' not supported between instances of 'int' and 'NoneType'
520 Test("test-invalid-client-hello-w-record-overflow.py"),
521
522 # Lots of failures. abrupt closure
523 Test("test-invalid-client-hello.py"),
524
525 # abrupt closure
526 # 'encrypted premaster set to all zero (n)' n in 256 384 512
527 Test("test-invalid-rsa-key-exchange-messages.py"),
528
529 # test expects illegal_parameter, we send unrecognized_name (which seems
530 # correct according to rfc 6066?)
531 Test("test-invalid-server-name-extension-resumption.py"),
532 # let through some server names without sending an alert
533 # again illegal_parameter vs unrecognized_name
534 Test("test-invalid-server-name-extension.py"),
535
536 # 4 failures:
537 # 'insecure (legacy) renegotiation with GET after 2nd handshake'
538 # 'insecure (legacy) renegotiation with incomplete GET'
539 # 'secure renegotiation with GET after 2nd handshake'
540 # 'secure renegotiation with incomplete GET'
541 Test("test-legacy-renegotiation.py"),
542
543 # 1 failure (timeout): we don't send the unexpected_message alert
544 # 'duplicate change cipher spec after Finished'
545 Test("test-message-duplication.py"),
546
547 # server should send status_request
548 Test("test-ocsp-stapling.py"),
549
550 # unexpected closure
551 Test("test-openssl-3712.py"),
552
553 # failed: 3 (expect an alert, we send AD)
554 # 'try insecure (legacy) renegotiation with incomplete GET'
555 # 'try secure renegotiation with GET after 2nd CH'
556 # 'try secure renegotiation with incomplete GET'
557 Test("test-renegotiation-disabled.py"),
558
559 # 'resumption of safe session with NULL cipher'
560 # 'resumption with cipher from old CH but not selected by server'
561 Test("test-resumption-with-wrong-ciphers.py"),
562
563 # 'session resumption with empty session_id'
564 # 'session resumption with random session_id'
565 # 'session resumption with renegotiation'
566 # AssertionError: Server did not send extension(s): session_ticket
567 Test("test-session-ticket-resumption.py"),
568
569 # 5 failures:
570 # 'empty sigalgs'
571 # 'only undefined sigalgs'
572 # 'rsa_pss_pss_sha256 only'
573 # 'rsa_pss_pss_sha384 only'
574 # 'rsa_pss_pss_sha512 only'
575 Test("test-sig-algs.py"),
576
577 # 13 failures:
578 # 'duplicated n non-rsa schemes' for n in 202 2342 8119 23741 32744
579 # 'empty list of signature methods'
580 # 'tolerance n RSA or ECDSA methods' for n in 215 2355 8132 23754
581 # 'tolerance 32758 methods with sig_alg_cert'
582 # 'tolerance max 32744 number of methods with sig_alg_cert'
583 # 'tolerance max (32760) number of methods'
584 Test("test-signature-algorithms.py"),
585
586 # times out
587 Test("test-ssl-death-alert.py"),
588
589 # 17 pass, 13 fail. padding and truncation
590 Test("test-truncating-of-client-hello.py"),
591
592 # x448 tests need disabling plus x25519 corner cases need sorting out
593 Test("test-x25519.py"),
594
595 # Needs TLS 1.0 or 1.1
596 Test("test-TLSv1_2-rejected-without-TLSv1_2.py"),
597])
598
599tls12_unsupported_tests = TestGroup("TLSv1.2 for unsupported features", [
600 # protocol_version
601 Test("test-SSLv3-padding.py"),
602 # we don't do RSA key exchanges
603 Test("test-bleichenbacher-timing.py"),
604 # no encrypt-then-mac
605 Test("test-encrypt-then-mac-renegotiation.py"),
606 Test("test-encrypt-then-mac.py"),
607 # no EME support
608 Test("test-extended-master-secret-extension-with-client-cert.py"),
609 Test("test-extended-master-secret-extension.py"),
610 # no ffdhe
611 Test("test-ffdhe-expected-params.py"),
612 Test("test-ffdhe-negotiation.py"),
613 # record_size_limit/max_fragment_length extension (RFC 8449)
614 Test("test-record-size-limit.py"),
615 # expects the server to send the heartbeat extension
616 Test("test-heartbeat.py"),
617 # needs an echo server
618 Test("test-lengths.py"),
619])
620
621# These tests take a ton of time to fail against an 1.3 server,
622# so don't run them against 1.3 pending further investigation.
623legacy_tests = TestGroup("Legacy protocol tests", [
624 Test("test-sslv2-force-cipher-3des.py"),
625 Test("test-sslv2-force-cipher-non3des.py"),
626 Test("test-sslv2-force-cipher.py"),
627 Test("test-sslv2-force-export-cipher.py"),
628 Test("test-sslv2hello-protocol.py"),
629])
630
631all_groups = [
632 tls13_tests,
633 tls13_slow_tests,
634 tls13_extra_cert_tests,
635 tls13_failing_tests,
636 tls13_slow_failing_tests,
637 tls13_unsupported_tests,
638 tls12_tests,
639 tls12_slow_tests,
640 tls12_failing_tests,
641 tls12_unsupported_tests,
642 legacy_tests,
643]
644
645failing_groups = [
646 tls13_failing_tests,
647 tls13_slow_failing_tests,
648 tls12_failing_tests,
649]
650
651class TestRunner:
652 """ Runs the given tests against a server and displays stats. """
653
654 def __init__(
655 self, timing=False, verbose=False, host="localhost", port=4433,
656 use_tls1_3=True, dry_run=False, tests=[], scriptdir=tlsfuzzer_scriptdir,
657 ):
658 self.tests = []
659
660 self.dryrun = dry_run
661 self.use_tls1_3 = use_tls1_3
662 self.host = host
663 self.port = str(port)
664 self.scriptdir = scriptdir
665
666 self.stats = []
667 self.failed = []
668 self.missing = []
669
670 self.timing = timing
671 self.verbose = verbose
672
673 def add(self, title="tests", tests=[]):
674 # tests.sort(key=lambda test: test.name)
675 self.tests.append(TestGroup(title, tests))
676
677 def add_group(self, group):
678 self.tests.append(group)
679
680 def run_script(self, test):
681 script = test.name
682 args = ["-h"] + [self.host] + ["-p"] + [self.port] + test.args(self.use_tls1_3)
683
684 if self.dryrun:
685 if not self.verbose:
686 args = []
687 print(script , end=' ' if args else '')
688 print(' '.join([f"\"{arg}\"" for arg in args]))
689 return
690
691 if self.verbose:
692 print(script)
693 else:
694 print(f"{script[:68]:<72}", end=" ", flush=True)
695 start = timer()
696 scriptpath = os.path.join(self.scriptdir, script)
697 if not os.path.exists(scriptpath):
698 self.missing.append(script)
699 print("MISSING")
700 return
701 test = subprocess.run(
702 ["python3", scriptpath] + args,
703 capture_output=not self.verbose,
704 text=True,
705 )
706 end = timer()
707 self.stats.append((script, end - start))
708 if test.returncode == 0:
709 print("OK")
710 return
711 print("FAILED")
712 self.failed.append(script)
713
714 if self.verbose:
715 return
716
717 print('\n'.join(test.stdout.split("Test end\n", 1)[1:]), end="")
718
719 def run(self):
720 for group in self:
721 print(f"Running {group.title} ...")
722 for test in group:
723 self.run_script(test)
724 return not self.failed
725
726 def __iter__(self):
727 return iter(self.tests)
728
729 def __del__(self):
730 if self.timing and self.stats:
731 total = 0.0
732 for (script, time) in self.stats:
733 print(f"{round(time, 2):6.2f} {script}")
734 total += time
735 print(f"{round(total, 2):6.2f} total")
736
737 if self.failed:
738 print("Failed tests:")
739 print('\n'.join(self.failed))
740
741 if self.missing:
742 print("Missing tests (outdated package?):")
743 print('\n'.join(self.missing))
744
745class TlsServer:
746 """ Spawns an s_server listening on localhost:port if necessary. """
747
748 def __init__(self, host="localhost", port=4433):
749 self.spawn = True
750 # Check whether a server is already listening on localhost:port
751 self.spawn = subprocess.run(
752 ["nc", "-c", "-z", "-T", "noverify", host, str(port)],
753 stderr=subprocess.DEVNULL,
754 ).returncode != 0
755
756 if self.spawn:
757 self.server = subprocess.Popen(
758 [
759 "openssl",
760 "s_server",
761 "-accept",
762 str(port),
763 "-groups",
764 "X25519:P-256:P-521:P-384",
765 "-key",
766 "localhost.key",
767 "-cert",
768 "localhost.crt",
769 "-www",
770 ],
771 stdout=subprocess.DEVNULL,
772 stderr=subprocess.PIPE,
773 text=True,
774 )
775
776 # Check whether the server talks TLSv1.3
777 self.has_tls1_3 = True or subprocess.run(
778 [
779 "nc",
780 "-c",
781 "-z",
782 "-T",
783 "noverify",
784 "-T",
785 "protocols=TLSv1.3",
786 "localhost",
787 str(port),
788 ],
789 stderr=subprocess.DEVNULL,
790 ).returncode == 0
791
792 self.check()
793
794 def check(self):
795 if self.spawn and self.server.poll() is not None:
796 print(self.server.stderr.read())
797 raise RuntimeError(
798 f"openssl s_server died. Return code: {self.server.returncode}."
799 )
800 if self.spawn:
801 self.server.stderr.detach()
802
803 def __del__(self):
804 if self.spawn:
805 self.server.terminate()
806
807# Extract the arguments we pass to script
808def defaultargs(script, has_tls1_3):
809 return next(
810 (test for group in all_groups for test in group if test.name == script),
811 Test()
812 ).args(has_tls1_3)
813
814def list_or_missing(missing=True):
815 tests = [test.name for group in all_groups for test in group]
816
817 if missing:
818 scripts = {
819 f for f in os.listdir(tlsfuzzer_scriptdir) if f != "__pycache__"
820 }
821 missing = scripts - set(tests)
822 if missing:
823 print('\n'.join(sorted(missing)))
824 exit(0)
825
826 tests.sort()
827 print('\n'.join(tests))
828 exit(0)
829
830def usage():
831 print("Usage: python3 tlsfuzzer.py [-flmnstv] [-p port] [script [test...]]")
832 print(" --help help")
833 print(" -f run failing tests")
834 print(" -l list tests")
835 print(" -m list new tests after package update")
836 print(" -n do not run tests, but list the ones that would be run")
837 print(" -p port connect to this port - defaults to 4433")
838 print(" -s run slow tests")
839 print(" -t show timing stats at end")
840 print(" -v verbose output")
841 exit(0)
842
843def main():
844 failing = False
845 list = False
846 missing = False
847 dryrun = False
848 host = "localhost"
849 port = 4433
850 slow = False
851 timing = False
852 verbose = False
853
854 argv = sys.argv[1:]
855 opts, args = getopt.getopt(argv, "fh:lmnp:stv", ["help"])
856 for opt, arg in opts:
857 if opt == '--help':
858 usage()
859 elif opt == '-f':
860 failing = True
861 elif opt == '-h':
862 host = arg
863 elif opt == '-l':
864 list = True
865 elif opt == '-m':
866 missing = True
867 elif opt == '-n':
868 dryrun = True
869 elif opt == '-p':
870 port = int(arg)
871 elif opt == '-s':
872 slow = True
873 elif opt == '-t':
874 timing = True
875 elif opt == '-v':
876 verbose = True
877 else:
878 raise ValueError(f"Unknown option: {opt}")
879
880 if not os.path.exists(tlsfuzzer_scriptdir):
881 print("package py3-tlsfuzzer is required for this regress")
882 exit(1)
883
884 if list and failing:
885 failing = [test.name for group in failing_groups for test in group]
886 failing.sort()
887 print('\n'.join(failing))
888 exit(0)
889
890 if list or missing:
891 list_or_missing(missing)
892
893 tls_server = TlsServer(host, port)
894
895 tests = TestRunner(timing, verbose, host, port, tls_server.has_tls1_3, dryrun)
896
897 if args:
898 (dir, script) = os.path.split(args[0])
899 if dir and not dir == '.':
900 tests.scriptdir = dir
901
902 testargs = defaultargs(script, tls_server.has_tls1_3)
903
904 tests.verbose = True
905 tests.add("test from command line", [Test(script, testargs + args[1:])])
906
907 exit(not tests.run())
908
909 if failing:
910 if tls_server.has_tls1_3:
911 tests.add_group(tls13_failing_tests)
912 if slow:
913 tests.add_group(tls13_slow_failing_tests)
914 tests.add_group(tls12_failing_tests)
915
916 if tls_server.has_tls1_3:
917 tests.add_group(tls13_tests)
918 if slow:
919 tests.add_group(tls13_slow_tests)
920 else:
921 tests.add_group(legacy_tests)
922
923 tests.add_group(tls12_tests)
924 if slow:
925 tests.add_group(tls12_slow_tests)
926
927 success = tests.run()
928 del tests
929
930 if not success:
931 print("FAILED")
932 exit(1)
933
934if __name__ == "__main__":
935 main()
generated by cgit v1.2.3-55-g6feb (git 2.39.1) at 2025-05-03 15:03:26 +0000