libssl: don't break TLSv1.2 with X25519MLKEM768

If the list of 'groups' starts with X25519MLKEM768 for a TLSv1.2 server,
ssl3_send_server_kex_ecdhe() attempts to use NID_X25519MLKEM768, which
it receives from tls1_get_supported_groups(). This does not work because
it never received the peer's public keys, which causes an error return
from tls_key_share_server_generate_mlkem768x25519().

For a TLSv1.2-only client with custom supported group list we will
currently send ML-KEM if configured. We should not do this.

There is more to fix here: if a TLSv1.2 client is misconfigured with
only X25519MLKEM768, we should not send a supported groups extension
(with this commit we'll send an empty one, which is an RFC violation).

This commit simply filters X25519MLKEM768 out of the supported groups
list if we're configured to be TLSv1.2-only.

feedback/ok jsing kenjiro (on an earlier version)

2 files changed, 30 insertions, 3 deletions

diff --git a/src/lib/libssl/ssl_tlsext.c b/src/lib/libssl/ssl_tlsext.c
index f6fbf43dfd..ccdb5d1dfa 100644
--- a/src/lib/libssl/ssl_tlsext.c
+++ b/src/lib/libssl/ssl_tlsext.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_tlsext.c,v 1.160 2026/05/09 11:45:50 tb Exp $ */
+/* $OpenBSD: ssl_tlsext.c,v 1.161 2026/06/06 08:45:41 tb Exp $ */
 /*
  * Copyright (c) 2016, 2017, 2019 Joel Sing <jsing@openbsd.org>
  * Copyright (c) 2017 Doug Hogan <doug@openbsd.org>
@@ -193,6 +193,10 @@ tlsext_alpn_client_process(SSL *s, uint16_t msg_type, CBS *cbs, int *alert)
 static int
 tlsext_supportedgroups_client_needs(SSL *s, uint16_t msg_type)
 {
+        /*
+         * XXX - Don't send an empty named_group_list. For TLSv1.3 we error
+         * earlier; for TLSv1.2 ensure we don't send the extension.
+         */
         return ssl_has_ecc_ciphers(s) ||
             (s->s3->hs.our_max_tls_version >= TLS1_3_VERSION);
 }
@@ -215,7 +219,7 @@ tlsext_supportedgroups_client_build(SSL *s, uint16_t msg_type, CBB *cbb)
                 return 0;
 
         for (i = 0; i < groups_len; i++) {
-                if (!ssl_security_supported_group(s, groups[i]))
+                if (!tls1_check_group(s, groups[i]))
                         continue;
                 if (!CBB_add_u16(&grouplist, groups[i]))
                         return 0;
diff --git a/src/lib/libssl/t1_lib.c b/src/lib/libssl/t1_lib.c
index 9947dcba1d..9dfcc0c0f3 100644
--- a/src/lib/libssl/t1_lib.c
+++ b/src/lib/libssl/t1_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: t1_lib.c,v 1.208 2026/06/04 18:02:52 tb Exp $ */
+/* $OpenBSD: t1_lib.c,v 1.209 2026/06/06 08:45:41 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -154,6 +154,7 @@ struct supported_group {
         uint16_t group_id;
         int nid;
         int bits;
+        uint16_t min_version;
 };
 
 /*
@@ -310,6 +311,7 @@ static const struct supported_group nid_list[] = {
                 .group_id = 4588,
                 .nid = NID_X25519MLKEM768,
                 .bits = 128,
+                .min_version = TLS1_3_VERSION,
         },
 };
 
@@ -512,6 +514,17 @@ tls1_group_id_present(uint16_t group_id, const uint16_t *list, size_t list_len)
         return 0;
 }
 
+static int
+tls1_group_id_allowed(const SSL *ssl, uint16_t group_id)
+{
+        const struct supported_group *sg;
+
+        if ((sg = tls1_supported_group_by_id(group_id)) == NULL)
+                return 0;
+
+        return ssl_effective_tls_version(ssl) >= sg->min_version;
+}
+
 int
 tls1_count_shared_groups(const SSL *ssl, size_t *out_count)
 {
@@ -529,6 +542,9 @@ tls1_count_shared_groups(const SSL *ssl, size_t *out_count)
                 if (!ssl_security_shared_group(ssl, pref[i]))
                         continue;
 
+                if (!tls1_group_id_allowed(ssl, pref[i]))
+                        continue;
+
                 count++;
         }
 
@@ -555,6 +571,9 @@ tls1_group_by_index(const SSL *ssl, size_t n, int *out_nid,
                 if (!ssl_security_fn(ssl, pref[i]))
                         continue;
 
+                if (!tls1_group_id_allowed(ssl, pref[i]))
+                        continue;
+
                 if (count++ == n)
                         return tls1_ec_group_id2nid(pref[i], out_nid);
         }
@@ -659,6 +678,10 @@ tls1_check_group(SSL *s, uint16_t group_id)
         for (i = 0; i < groupslen; i++) {
                 if (!ssl_security_supported_group(s, groups[i]))
                         continue;
+
+                if (!tls1_group_id_allowed(s, groups[i]))
+                        continue;
+
                 if (groups[i] == group_id)
                         return 1;
         }